2024-04-23T00:30:39.057 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-23T00:30:39.057 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi --verbose --description orch/cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --archive /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995 --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/orig.config.yaml 2024-04-23T00:30:39.094 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-23T00:30:44.710 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995 branch: reef description: orch/cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7668995' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 646b8c7d293d8ec31a40c69f7b3237b19c1be24d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 646b8c7d293d8ec31a40c69f7b3237b19c1be24d selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: f26c9c3fee57ca330501910de5a07c8769ef5dfc owner: scheduled_teuthology@teuthology priority: 930 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 4115 sha1: 646b8c7d293d8ec31a40c69f7b3237b19c1be24d sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_f26c9c3fee57ca330501910de5a07c8769ef5dfc/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: f26c9c3fee57ca330501910de5a07c8769ef5dfc targets: smithi121.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXcXgtI4HvLZYXDYnNW+92ABIvDnjr8IcGJ8aNWp8V8DIm4z/kyJ5PQ93PXoShH/WV0QwkvP8o0HempSCyXbNI= smithi153.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLHM3gVziTYYkCCKsYnfZAti80V9AhZxA5ZWKcMCBvtdPoxW18QVNOVZ1QxDDYrjooxoCwOW9iRQWZt7W5zduoY= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - cephadm: avoid_pacific_features: true cephadm_branch: v16.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v16.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - ceph config get mgr mgr/cephadm/migration_current | grep 6 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_22:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-23T00:30:44.710 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_f26c9c3fee57ca330501910de5a07c8769ef5dfc/qa; will attempt to use it 2024-04-23T00:30:44.711 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_f26c9c3fee57ca330501910de5a07c8769ef5dfc/qa/tasks 2024-04-23T00:30:44.711 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-23T00:30:44.712 INFO:teuthology.task.internal:Checking packages... 2024-04-23T00:30:44.740 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '646b8c7d293d8ec31a40c69f7b3237b19c1be24d' 2024-04-23T00:30:44.740 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-23T00:30:44.740 INFO:teuthology.packaging:ref: None 2024-04-23T00:30:44.740 INFO:teuthology.packaging:tag: None 2024-04-23T00:30:44.740 INFO:teuthology.packaging:branch: reef 2024-04-23T00:30:44.740 INFO:teuthology.packaging:sha1: 646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:30:44.741 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-23T00:30:45.039 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-0 2024-04-23T00:30:45.040 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-23T00:30:45.046 INFO:teuthology.task.internal:no buildpackages task found 2024-04-23T00:30:45.046 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-23T00:30:45.078 INFO:teuthology.task.internal:Saving configuration 2024-04-23T00:30:45.092 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-23T00:30:45.103 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-23T00:30:45.135 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi121.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-23 00:25:25.097190', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXcXgtI4HvLZYXDYnNW+92ABIvDnjr8IcGJ8aNWp8V8DIm4z/kyJ5PQ93PXoShH/WV0QwkvP8o0HempSCyXbNI='} 2024-04-23T00:30:45.157 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi153.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-23 00:25:25.095870', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLHM3gVziTYYkCCKsYnfZAti80V9AhZxA5ZWKcMCBvtdPoxW18QVNOVZ1QxDDYrjooxoCwOW9iRQWZt7W5zduoY='} 2024-04-23T00:30:45.157 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-23T00:30:45.162 INFO:teuthology.task.internal:roles: ubuntu@smithi121.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-23T00:30:45.162 INFO:teuthology.task.internal:roles: ubuntu@smithi153.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-23T00:30:45.162 INFO:teuthology.run_tasks:Running task console_log... 2024-04-23T00:30:45.269 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff332f73f70>, signals=[15]) 2024-04-23T00:30:45.269 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-23T00:30:45.275 INFO:teuthology.task.internal:Opening connections... 2024-04-23T00:30:45.275 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:30:45.277 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:30:45.358 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi153.front.sepia.ceph.com 2024-04-23T00:30:45.359 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:30:45.430 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-23T00:30:45.437 DEBUG:teuthology.orchestra.run.smithi121:> uname -m 2024-04-23T00:30:45.461 INFO:teuthology.orchestra.run.smithi121.stdout:x86_64 2024-04-23T00:30:45.462 DEBUG:teuthology.orchestra.run.smithi121:> cat /etc/os-release 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:NAME="CentOS Stream" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION="8" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:ID="centos" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:ID_LIKE="rhel fedora" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:VERSION_ID="8" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:PLATFORM_ID="platform:el8" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:ANSI_COLOR="0;31" 2024-04-23T00:30:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-23T00:30:45.524 INFO:teuthology.orchestra.run.smithi121.stdout:HOME_URL="https://centos.org/" 2024-04-23T00:30:45.524 INFO:teuthology.orchestra.run.smithi121.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-23T00:30:45.524 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-23T00:30:45.524 INFO:teuthology.orchestra.run.smithi121.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-23T00:30:45.525 INFO:teuthology.lock.ops:Updating smithi121.front.sepia.ceph.com on lock server 2024-04-23T00:30:49.878 DEBUG:teuthology.orchestra.run.smithi153:> uname -m 2024-04-23T00:30:49.901 INFO:teuthology.orchestra.run.smithi153.stdout:x86_64 2024-04-23T00:30:49.901 DEBUG:teuthology.orchestra.run.smithi153:> cat /etc/os-release 2024-04-23T00:30:49.962 INFO:teuthology.orchestra.run.smithi153.stdout:NAME="CentOS Stream" 2024-04-23T00:30:49.962 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION="8" 2024-04-23T00:30:49.962 INFO:teuthology.orchestra.run.smithi153.stdout:ID="centos" 2024-04-23T00:30:49.962 INFO:teuthology.orchestra.run.smithi153.stdout:ID_LIKE="rhel fedora" 2024-04-23T00:30:49.962 INFO:teuthology.orchestra.run.smithi153.stdout:VERSION_ID="8" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:PLATFORM_ID="platform:el8" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:ANSI_COLOR="0;31" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:HOME_URL="https://centos.org/" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-23T00:30:49.963 INFO:teuthology.orchestra.run.smithi153.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-23T00:30:49.963 INFO:teuthology.lock.ops:Updating smithi153.front.sepia.ceph.com on lock server 2024-04-23T00:30:49.982 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-23T00:30:49.995 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-23T00:30:50.019 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-23T00:30:50.020 DEBUG:teuthology.orchestra.run.smithi121:> test '!' -e /home/ubuntu/cephtest 2024-04-23T00:30:50.022 DEBUG:teuthology.orchestra.run.smithi153:> test '!' -e /home/ubuntu/cephtest 2024-04-23T00:30:50.042 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-23T00:30:50.070 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-23T00:30:50.070 DEBUG:teuthology.orchestra.run.smithi121:> test -z $(ls -A /var/lib/ceph) 2024-04-23T00:30:50.084 DEBUG:teuthology.orchestra.run.smithi153:> test -z $(ls -A /var/lib/ceph) 2024-04-23T00:30:50.121 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-23T00:30:50.262 INFO:teuthology.run_tasks:Running task kernel... 2024-04-23T00:30:50.277 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-23T00:30:50.278 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-23T00:30:50.278 DEBUG:teuthology.orchestra.run.smithi121:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-23T00:30:50.278 DEBUG:teuthology.orchestra.run.smithi153:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-23T00:30:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T00:30:50.298 DEBUG:teuthology.orchestra.run.smithi153:> uname -r 2024-04-23T00:30:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T00:30:50.299 DEBUG:teuthology.orchestra.run.smithi121:> uname -r 2024-04-23T00:30:50.359 INFO:teuthology.orchestra.run.smithi153.stdout:4.18.0-489.el8.x86_64 2024-04-23T00:30:50.359 INFO:teuthology.task.kernel:Running kernel on smithi153: 4.18.0-489.el8.x86_64 2024-04-23T00:30:50.359 DEBUG:teuthology.orchestra.run.smithi153:> sudo yum install -y kernel 2024-04-23T00:30:50.361 INFO:teuthology.orchestra.run.smithi121.stdout:4.18.0-489.el8.x86_64 2024-04-23T00:30:50.361 INFO:teuthology.task.kernel:Running kernel on smithi121: 4.18.0-489.el8.x86_64 2024-04-23T00:30:50.361 DEBUG:teuthology.orchestra.run.smithi121:> sudo yum install -y kernel 2024-04-23T00:30:54.904 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS Stream 8 - AppStream 14 MB/s | 28 MB 00:02 2024-04-23T00:30:54.974 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 8 - AppStream 14 MB/s | 28 MB 00:02 2024-04-23T00:31:00.913 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS Stream 8 - BaseOS 10 MB/s | 10 MB 00:01 2024-04-23T00:31:00.982 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 8 - BaseOS 10 MB/s | 10 MB 00:00 2024-04-23T00:31:02.840 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 8 - Extras 82 kB/s | 18 kB 00:00 2024-04-23T00:31:02.946 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS Stream 8 - Extras 43 kB/s | 18 kB 00:00 2024-04-23T00:31:03.324 INFO:teuthology.orchestra.run.smithi121.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 7.7 kB 00:00 2024-04-23T00:31:03.567 INFO:teuthology.orchestra.run.smithi121.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2024-04-23T00:31:03.697 INFO:teuthology.orchestra.run.smithi153.stdout:CentOS Stream 8 - Extras common packages 12 kB/s | 7.7 kB 00:00 2024-04-23T00:31:03.931 INFO:teuthology.orchestra.run.smithi153.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2024-04-23T00:31:04.893 INFO:teuthology.orchestra.run.smithi121.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2024-04-23T00:31:05.401 INFO:teuthology.orchestra.run.smithi153.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2024-04-23T00:31:08.645 INFO:teuthology.orchestra.run.smithi121.stdout:lab-extras 493 kB/s | 24 kB 00:00 2024-04-23T00:31:09.174 INFO:teuthology.orchestra.run.smithi153.stdout:lab-extras 500 kB/s | 24 kB 00:00 2024-04-23T00:31:11.706 INFO:teuthology.orchestra.run.smithi121.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-23T00:31:11.706 INFO:teuthology.orchestra.run.smithi121.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout:Installing: 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-23T00:31:11.831 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:Install 3 Packages 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 90 M 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 96 M 2024-04-23T00:31:11.832 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2024-04-23T00:31:12.153 INFO:teuthology.orchestra.run.smithi153.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-23T00:31:12.153 INFO:teuthology.orchestra.run.smithi153.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-23T00:31:12.278 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2024-04-23T00:31:12.279 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:31:12.279 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2024-04-23T00:31:12.279 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout:Installing: 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout:Install 3 Packages 2024-04-23T00:31:12.280 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:12.281 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 90 M 2024-04-23T00:31:12.281 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 96 M 2024-04-23T00:31:12.281 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2024-04-23T00:31:13.045 INFO:teuthology.orchestra.run.smithi121.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 10 MB/s | 10 MB 00:01 2024-04-23T00:31:13.441 INFO:teuthology.orchestra.run.smithi153.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 11 MB/s | 10 MB 00:00 2024-04-23T00:31:13.961 INFO:teuthology.orchestra.run.smithi121.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 19 MB/s | 36 MB 00:01 2024-04-23T00:31:14.433 INFO:teuthology.orchestra.run.smithi153.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 19 MB/s | 36 MB 00:01 2024-04-23T00:31:14.486 INFO:teuthology.orchestra.run.smithi121.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 18 MB/s | 44 MB 00:02 2024-04-23T00:31:14.486 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2024-04-23T00:31:14.486 INFO:teuthology.orchestra.run.smithi121.stdout:Total 34 MB/s | 90 MB 00:02 2024-04-23T00:31:14.967 INFO:teuthology.orchestra.run.smithi153.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 18 MB/s | 44 MB 00:02 2024-04-23T00:31:14.967 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2024-04-23T00:31:14.968 INFO:teuthology.orchestra.run.smithi153.stdout:Total 34 MB/s | 90 MB 00:02 2024-04-23T00:31:15.170 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2024-04-23T00:31:15.588 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2024-04-23T00:31:15.588 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2024-04-23T00:31:15.650 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2024-04-23T00:31:16.068 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2024-04-23T00:31:16.068 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2024-04-23T00:31:16.825 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2024-04-23T00:31:16.841 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2024-04-23T00:31:17.310 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2024-04-23T00:31:17.342 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2024-04-23T00:31:19.608 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2024-04-23T00:31:20.128 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:20.138 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2024-04-23T00:31:20.733 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:23.661 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:24.069 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:24.356 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:24.716 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:27.504 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:27.768 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:28.151 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:28.424 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:49.753 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:49.778 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.126 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.126 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:50.126 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:50.304 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.329 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.623 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.623 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:50.623 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2024-04-23T00:31:50.623 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-23T00:31:50.623 INFO:teuthology.orchestra.run.smithi121.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-23T00:31:50.624 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:50.624 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2024-04-23T00:31:50.678 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:50.678 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-23T00:31:50.678 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-23T00:31:50.772 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q kernel | sort -rV | head -n 1 2024-04-23T00:31:50.986 INFO:teuthology.orchestra.run.smithi121.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-23T00:31:50.986 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-23T00:31:50.987 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-23T00:31:50.987 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-23T00:31:50.987 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-23T00:31:50.987 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-23T00:31:50.987 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi121.front.sepia.ceph.com, path=None, version=distro) 2024-04-23T00:31:50.987 DEBUG:teuthology.orchestra.run.smithi121:> rpm -q kernel | sort -rV | head -n 1 2024-04-23T00:31:51.198 INFO:teuthology.orchestra.run.smithi121.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-23T00:31:51.198 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.199 DEBUG:teuthology.orchestra.run.smithi121:> sudo rpm -qi grub2-tools 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:51.200 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Name : grub2-tools 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Epoch : 1 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Version : 2.02 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Release : 129.el8 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Architecture: x86_64 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Group : System Environment/Base 2024-04-23T00:31:51.232 INFO:teuthology.orchestra.run.smithi121.stdout:Size : 9265586 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:License : GPLv3+ 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Relocations : (not relocatable) 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Packager : CentOS BuildSystem 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Vendor : CentOS 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Summary : Support tools for GRUB. 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:Description : 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-23T00:31:51.233 INFO:teuthology.orchestra.run.smithi121.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-23T00:31:51.234 INFO:teuthology.orchestra.run.smithi121.stdout:hardware devices. 2024-04-23T00:31:51.234 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:31:51.234 INFO:teuthology.orchestra.run.smithi121.stdout:This subpackage provides tools for support of all platforms. 2024-04-23T00:31:51.234 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-23T00:31:51.235 INFO:teuthology.task.kernel:Updating grub on smithi121 to boot 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-23T00:31:51.366 DEBUG:teuthology.orchestra.run.smithi153:> rpm -q kernel | sort -rV | head -n 1 2024-04-23T00:31:51.585 INFO:teuthology.orchestra.run.smithi153.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-23T00:31:51.585 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.586 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.586 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-23T00:31:51.586 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-23T00:31:51.586 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-23T00:31:51.586 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi153.front.sepia.ceph.com, path=None, version=distro) 2024-04-23T00:31:51.586 DEBUG:teuthology.orchestra.run.smithi153:> rpm -q kernel | sort -rV | head -n 1 2024-04-23T00:31:51.799 INFO:teuthology.orchestra.run.smithi153.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-23T00:31:51.799 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.799 DEBUG:teuthology.orchestra.run.smithi153:> sudo rpm -qi grub2-tools 2024-04-23T00:31:51.833 INFO:teuthology.orchestra.run.smithi153.stdout:Name : grub2-tools 2024-04-23T00:31:51.833 INFO:teuthology.orchestra.run.smithi153.stdout:Epoch : 1 2024-04-23T00:31:51.833 INFO:teuthology.orchestra.run.smithi153.stdout:Version : 2.02 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Release : 129.el8 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Architecture: x86_64 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Group : System Environment/Base 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Size : 9265586 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:License : GPLv3+ 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Relocations : (not relocatable) 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Packager : CentOS BuildSystem 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:Vendor : CentOS 2024-04-23T00:31:51.834 INFO:teuthology.orchestra.run.smithi153.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:Summary : Support tools for GRUB. 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:Description : 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:hardware devices. 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:31:51.835 INFO:teuthology.orchestra.run.smithi153.stdout:This subpackage provides tools for support of all platforms. 2024-04-23T00:31:51.836 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-23T00:31:51.836 INFO:teuthology.task.kernel:Updating grub on smithi153 to boot 4.18.0-552.el8.x86_64 2024-04-23T00:31:51.836 DEBUG:teuthology.orchestra.run.smithi153:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-23T00:31:52.915 INFO:teuthology.orchestra.run.smithi121.stderr:Generating grub configuration file ... 2024-04-23T00:31:53.291 INFO:teuthology.orchestra.run.smithi153.stderr:Generating grub configuration file ... 2024-04-23T00:31:54.918 INFO:teuthology.orchestra.run.smithi121.stderr:done 2024-04-23T00:31:54.919 DEBUG:teuthology.orchestra.run.smithi121:> mktemp 2024-04-23T00:31:54.960 INFO:teuthology.orchestra.run.smithi121.stdout:/tmp/tmp.gktrkiRTwy 2024-04-23T00:31:54.960 DEBUG:teuthology.orchestra.run.smithi121:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.gktrkiRTwy 2024-04-23T00:31:55.043 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 0666 /tmp/tmp.gktrkiRTwy 2024-04-23T00:31:55.187 DEBUG:teuthology.orchestra.remote:smithi121:/tmp/tmp.gktrkiRTwy is 6KB 2024-04-23T00:31:55.245 DEBUG:teuthology.orchestra.run.smithi121:> rm -fr /tmp/tmp.gktrkiRTwy 2024-04-23T00:31:55.262 DEBUG:teuthology.orchestra.run.smithi121:> sudo /bin/ls /boot/loader/entries || true 2024-04-23T00:31:55.337 INFO:teuthology.orchestra.run.smithi121.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-23T00:31:55.337 INFO:teuthology.orchestra.run.smithi121.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-23T00:31:55.337 INFO:teuthology.orchestra.run.smithi121.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-23T00:31:55.337 INFO:teuthology.orchestra.run.smithi121.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-23T00:31:55.338 DEBUG:teuthology.orchestra.run.smithi121:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-23T00:31:55.364 INFO:teuthology.orchestra.run.smithi153.stderr:done 2024-04-23T00:31:55.365 DEBUG:teuthology.orchestra.run.smithi153:> mktemp 2024-04-23T00:31:55.393 INFO:teuthology.orchestra.run.smithi153.stdout:/tmp/tmp.2ww90nM5jR 2024-04-23T00:31:55.394 DEBUG:teuthology.orchestra.run.smithi153:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.2ww90nM5jR 2024-04-23T00:31:55.467 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 0666 /tmp/tmp.2ww90nM5jR 2024-04-23T00:31:55.501 DEBUG:teuthology.orchestra.run.smithi121:> sudo shutdown -r now 2024-04-23T00:31:55.620 DEBUG:teuthology.orchestra.remote:smithi153:/tmp/tmp.2ww90nM5jR is 6KB 2024-04-23T00:31:55.668 DEBUG:teuthology.orchestra.run.smithi153:> rm -fr /tmp/tmp.2ww90nM5jR 2024-04-23T00:31:55.686 DEBUG:teuthology.orchestra.run.smithi153:> sudo /bin/ls /boot/loader/entries || true 2024-04-23T00:31:55.751 INFO:teuthology.orchestra.run.smithi153.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-23T00:31:55.751 INFO:teuthology.orchestra.run.smithi153.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-23T00:31:55.751 INFO:teuthology.orchestra.run.smithi153.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-23T00:31:55.751 INFO:teuthology.orchestra.run.smithi153.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-23T00:31:55.752 DEBUG:teuthology.orchestra.run.smithi153:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-23T00:31:55.916 DEBUG:teuthology.orchestra.run.smithi153:> sudo shutdown -r now 2024-04-23T00:32:25.530 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-23T00:32:25.531 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-23T00:32:25.532 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:32:25.919 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-23T00:32:25.919 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:32:25.920 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:33:00.447 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.153 2024-04-23T00:33:09.454 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:33:09.455 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:33:25.551 DEBUG:teuthology.orchestra.remote:timed out 2024-04-23T00:33:27.838 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.153 2024-04-23T00:33:34.558 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-23T00:33:34.559 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:33:34.995 DEBUG:teuthology.orchestra.run.smithi121:> true 2024-04-23T00:33:35.204 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-23T00:33:35.204 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-552.el8.x86_64"... 2024-04-23T00:33:35.205 DEBUG:teuthology.orchestra.run.smithi121:> uname -r 2024-04-23T00:33:35.261 INFO:teuthology.orchestra.run.smithi121.stdout:4.18.0-552.el8.x86_64 2024-04-23T00:33:35.261 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-23T00:33:35.261 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-23T00:33:35.261 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-23T00:33:36.262 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-23T00:33:36.266 DEBUG:teuthology.orchestra.run.smithi121:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-23T00:33:36.326 INFO:teuthology.orchestra.run.smithi121.stdout:ttyS1 2024-04-23T00:33:36.339 DEBUG:teuthology.parallel:result is None 2024-04-23T00:33:39.839 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:33:39.840 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:33:41.269 DEBUG:teuthology.orchestra.run.smithi153:> true 2024-04-23T00:33:41.705 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:33:41.705 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "4.18.0-552.el8.x86_64"... 2024-04-23T00:33:41.705 DEBUG:teuthology.orchestra.run.smithi153:> uname -r 2024-04-23T00:33:41.723 INFO:teuthology.orchestra.run.smithi153.stdout:4.18.0-552.el8.x86_64 2024-04-23T00:33:41.724 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-23T00:33:41.724 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-23T00:33:41.724 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-23T00:33:42.725 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-23T00:33:42.725 DEBUG:teuthology.orchestra.run.smithi153:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-23T00:33:42.849 INFO:teuthology.orchestra.run.smithi153.stdout:ttyS1 2024-04-23T00:33:42.861 DEBUG:teuthology.parallel:result is None 2024-04-23T00:33:42.861 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-23T00:33:42.869 INFO:teuthology.task.internal:Creating test directory... 2024-04-23T00:33:42.869 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-23T00:33:42.872 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-23T00:33:42.891 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-23T00:33:42.896 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-23T00:33:42.928 INFO:teuthology.task.internal:Creating archive directory... 2024-04-23T00:33:42.929 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-23T00:33:42.933 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-23T00:33:42.958 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-23T00:33:42.964 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-23T00:33:42.965 DEBUG:teuthology.orchestra.run.smithi121:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-23T00:33:42.996 DEBUG:teuthology.orchestra.run.smithi153:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-23T00:33:43.026 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-23T00:33:43.036 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-23T00:33:43.037 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-23T00:33:43.047 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-23T00:33:43.048 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-23T00:33:43.055 INFO:teuthology.task.internal:Configuring sudo... 2024-04-23T00:33:43.056 DEBUG:teuthology.orchestra.run.smithi121:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-23T00:33:43.080 DEBUG:teuthology.orchestra.run.smithi153:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-23T00:33:43.116 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-23T00:33:43.126 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-23T00:33:43.126 DEBUG:teuthology.orchestra.run.smithi121:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-23T00:33:43.148 DEBUG:teuthology.orchestra.run.smithi153:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-23T00:33:43.173 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-23T00:33:43.255 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-23T00:33:43.303 DEBUG:teuthology.orchestra.run.smithi121:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-23T00:33:43.390 DEBUG:teuthology.orchestra.run.smithi121:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-23T00:33:43.428 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:33:43.428 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-23T00:33:43.493 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-23T00:33:43.538 DEBUG:teuthology.orchestra.run.smithi153:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-23T00:33:43.593 DEBUG:teuthology.orchestra.run.smithi153:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-23T00:33:43.683 DEBUG:teuthology.orchestra.run.smithi153:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-23T00:33:43.720 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:33:43.720 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-23T00:33:43.787 DEBUG:teuthology.orchestra.run.smithi121:> sudo service rsyslog restart 2024-04-23T00:33:43.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo service rsyslog restart 2024-04-23T00:33:43.844 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-23T00:33:43.892 INFO:teuthology.orchestra.run.smithi153.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-23T00:33:44.322 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-23T00:33:44.330 INFO:teuthology.task.internal:Starting timer... 2024-04-23T00:33:44.330 INFO:teuthology.run_tasks:Running task pcp... 2024-04-23T00:33:44.358 INFO:teuthology.run_tasks:Running task selinux... 2024-04-23T00:33:44.391 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-23T00:33:44.392 DEBUG:teuthology.orchestra.run.smithi121:> sudo service auditd rotate 2024-04-23T00:33:44.484 INFO:teuthology.orchestra.run.smithi121.stdout:Rotating logs: 2024-04-23T00:33:44.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo service auditd rotate 2024-04-23T00:33:44.608 INFO:teuthology.orchestra.run.smithi153.stdout:Rotating logs: 2024-04-23T00:33:44.609 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-23T00:33:44.610 DEBUG:teuthology.orchestra.run.smithi121:> /usr/sbin/getenforce 2024-04-23T00:33:44.642 INFO:teuthology.orchestra.run.smithi121.stdout:Permissive 2024-04-23T00:33:44.642 DEBUG:teuthology.orchestra.run.smithi153:> /usr/sbin/getenforce 2024-04-23T00:33:44.667 INFO:teuthology.orchestra.run.smithi153.stdout:Permissive 2024-04-23T00:33:44.667 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi121.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi153.front.sepia.ceph.com': 'permissive'} 2024-04-23T00:33:44.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-23T00:33:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T00:33:44.709 DEBUG:teuthology.orchestra.run.smithi153:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-23T00:33:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T00:33:44.735 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-23T00:33:44.735 DEBUG:teuthology.orchestra.run.smithi121:> sudo /usr/sbin/setenforce permissive 2024-04-23T00:33:44.775 DEBUG:teuthology.orchestra.run.smithi153:> sudo /usr/sbin/setenforce permissive 2024-04-23T00:33:44.802 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-23T00:33:44.812 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-23T00:33:44.817 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-23T00:33:44.949 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-23T00:33:44.966 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-23T00:33:44.967 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi121.front.sepia.ceph.com,smithi153.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-23T00:39:38.020 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi121.front.sepia.ceph.com'), Remote(name='ubuntu@smithi153.front.sepia.ceph.com')] 2024-04-23T00:39:38.022 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-23T00:39:38.023 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi121.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:39:38.104 DEBUG:teuthology.orchestra.run.smithi121:> true 2024-04-23T00:39:38.173 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi121.front.sepia.ceph.com' 2024-04-23T00:39:38.174 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:39:38.174 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi153.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-23T00:39:38.248 DEBUG:teuthology.orchestra.run.smithi153:> true 2024-04-23T00:39:38.320 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi153.front.sepia.ceph.com' 2024-04-23T00:39:38.321 INFO:teuthology.run_tasks:Running task clock... 2024-04-23T00:39:38.339 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-23T00:39:38.339 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-23T00:39:38.340 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-23T00:39:38.342 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-23T00:39:38.342 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-23T00:39:38.371 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-23T00:39:38.385 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-23T00:39:38.392 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-23T00:39:38.406 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-23T00:39:38.452 INFO:teuthology.orchestra.run.smithi121.stderr:sudo: ntpd: command not found 2024-04-23T00:39:38.462 INFO:teuthology.orchestra.run.smithi121.stdout:506 Cannot talk to daemon 2024-04-23T00:39:38.476 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-23T00:39:38.481 INFO:teuthology.orchestra.run.smithi153.stderr:sudo: ntpd: command not found 2024-04-23T00:39:38.488 INFO:teuthology.orchestra.run.smithi121.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-23T00:39:38.492 INFO:teuthology.orchestra.run.smithi153.stdout:506 Cannot talk to daemon 2024-04-23T00:39:38.506 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-23T00:39:38.519 INFO:teuthology.orchestra.run.smithi153.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-23T00:39:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:bash: ntpq: command not found 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.556 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.580 INFO:teuthology.orchestra.run.smithi153.stderr:bash: ntpq: command not found 2024-04-23T00:39:38.583 INFO:teuthology.orchestra.run.smithi153.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-23T00:39:38.583 INFO:teuthology.orchestra.run.smithi153.stdout:=============================================================================== 2024-04-23T00:39:38.583 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.583 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.583 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.584 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T00:39:38.584 INFO:teuthology.run_tasks:Running task pexec... 2024-04-23T00:39:38.594 INFO:teuthology.task.pexec:Executing custom commands... 2024-04-23T00:39:38.594 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:39:38.595 DEBUG:teuthology.orchestra.run.smithi121:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-23T00:39:38.595 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi153.front.sepia.ceph.com 2024-04-23T00:39:38.595 DEBUG:teuthology.orchestra.run.smithi153:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-23T00:39:39.356 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:24 ago on Tue 23 Apr 2024 12:38:15 AM UTC. 2024-04-23T00:39:39.384 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:23 ago on Tue 23 Apr 2024 12:38:16 AM UTC. 2024-04-23T00:39:41.493 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout: Package Architecture Version Repository Size 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout:Resetting modules: 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:41.494 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:41.532 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout: Package Architecture Version Repository Size 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout:Resetting modules: 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:41.533 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:41.739 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2024-04-23T00:39:41.789 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2024-04-23T00:39:42.518 INFO:teuthology.orchestra.run.smithi121.stdout:Last metadata expiration check: 0:01:27 ago on Tue 23 Apr 2024 12:38:15 AM UTC. 2024-04-23T00:39:42.583 INFO:teuthology.orchestra.run.smithi153.stdout:Last metadata expiration check: 0:01:26 ago on Tue 23 Apr 2024 12:38:16 AM UTC. 2024-04-23T00:39:44.633 INFO:teuthology.orchestra.run.smithi121.stdout:Dependencies resolved. 2024-04-23T00:39:44.670 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:44.670 INFO:teuthology.orchestra.run.smithi121.stdout: Package Arch Version Repository Size 2024-04-23T00:39:44.670 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout:Installing group/module packages: 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: aardvark-dns x86_64 2:1.10.0-1.module_el8+871+552be958 appstream 1.0 M 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: buildah x86_64 2:1.33.6-2.module_el8+976+11360731 appstream 9.6 M 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: cockpit-podman noarch 84.1-1.module_el8+977+8ab54275 appstream 689 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: crun x86_64 1.14.3-2.module_el8+968+fbb249c7 appstream 258 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: netavark x86_64 2:1.10.3-1.module_el8+951+32019cde appstream 4.1 M 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: python3-podman noarch 4.9.0-1.module_el8+847+7863d4e6 appstream 180 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: skopeo x86_64 2:1.14.3-0.1.module_el8+885+7da147f3 appstream 8.8 M 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: toolbox x86_64 0.0.99.5-2.module_el8+971+3d3df00d appstream 2.5 M 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: udica noarch 0.2.6-20.module_el8+804+f131391c appstream 53 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout:Installing dependencies: 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2024-04-23T00:39:44.671 INFO:teuthology.orchestra.run.smithi121.stdout:Installing weak dependencies: 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout:Installing module profiles: 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools/common 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout:Enabling module streams: 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: container-tools rhel8 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction Summary 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout:================================================================================ 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout:Install 13 Packages 2024-04-23T00:39:44.672 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:44.673 INFO:teuthology.orchestra.run.smithi121.stdout:Total download size: 27 M 2024-04-23T00:39:44.673 INFO:teuthology.orchestra.run.smithi121.stdout:Installed size: 91 M 2024-04-23T00:39:44.673 INFO:teuthology.orchestra.run.smithi121.stdout:Downloading Packages: 2024-04-23T00:39:44.713 INFO:teuthology.orchestra.run.smithi153.stdout:Dependencies resolved. 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: Package Arch Version Repository Size 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout:Installing group/module packages: 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: aardvark-dns x86_64 2:1.10.0-1.module_el8+871+552be958 appstream 1.0 M 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: buildah x86_64 2:1.33.6-2.module_el8+976+11360731 appstream 9.6 M 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: cockpit-podman noarch 84.1-1.module_el8+977+8ab54275 appstream 689 k 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: crun x86_64 1.14.3-2.module_el8+968+fbb249c7 appstream 258 k 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: netavark x86_64 2:1.10.3-1.module_el8+951+32019cde appstream 4.1 M 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: python3-podman noarch 4.9.0-1.module_el8+847+7863d4e6 appstream 180 k 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: skopeo x86_64 2:1.14.3-0.1.module_el8+885+7da147f3 appstream 8.8 M 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: toolbox x86_64 0.0.99.5-2.module_el8+971+3d3df00d appstream 2.5 M 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout: udica noarch 0.2.6-20.module_el8+804+f131391c appstream 53 k 2024-04-23T00:39:44.751 INFO:teuthology.orchestra.run.smithi153.stdout:Installing dependencies: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:Installing weak dependencies: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:Installing module profiles: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools/common 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:Enabling module streams: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: container-tools rhel8 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction Summary 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:================================================================================ 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout:Install 13 Packages 2024-04-23T00:39:44.752 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:44.753 INFO:teuthology.orchestra.run.smithi153.stdout:Total download size: 27 M 2024-04-23T00:39:44.753 INFO:teuthology.orchestra.run.smithi153.stdout:Installed size: 91 M 2024-04-23T00:39:44.753 INFO:teuthology.orchestra.run.smithi153.stdout:Downloading Packages: 2024-04-23T00:39:45.360 INFO:teuthology.orchestra.run.smithi153.stdout:(1/13): cockpit-podman-84.1-1.module_el8+977+8a 1.2 MB/s | 689 kB 00:00 2024-04-23T00:39:45.418 INFO:teuthology.orchestra.run.smithi153.stdout:(2/13): aardvark-dns-1.10.0-1.module_el8+871+55 1.6 MB/s | 1.0 MB 00:00 2024-04-23T00:39:45.422 INFO:teuthology.orchestra.run.smithi121.stdout:(1/13): cockpit-podman-84.1-1.module_el8+977+8a 1.2 MB/s | 689 kB 00:00 2024-04-23T00:39:45.460 INFO:teuthology.orchestra.run.smithi153.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 378 kB/s | 38 kB 00:00 2024-04-23T00:39:45.519 INFO:teuthology.orchestra.run.smithi153.stdout:(4/13): crun-1.14.3-2.module_el8+968+fbb249c7.x 2.5 MB/s | 258 kB 00:00 2024-04-23T00:39:45.523 INFO:teuthology.orchestra.run.smithi121.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 379 kB/s | 38 kB 00:00 2024-04-23T00:39:45.556 INFO:teuthology.orchestra.run.smithi121.stdout:(3/13): aardvark-dns-1.10.0-1.module_el8+871+55 1.5 MB/s | 1.0 MB 00:00 2024-04-23T00:39:45.623 INFO:teuthology.orchestra.run.smithi121.stdout:(4/13): crun-1.14.3-2.module_el8+968+fbb249c7.x 2.5 MB/s | 258 kB 00:00 2024-04-23T00:39:45.644 INFO:teuthology.orchestra.run.smithi153.stdout:(5/13): python3-podman-4.9.0-1.module_el8+847+7 1.4 MB/s | 180 kB 00:00 2024-04-23T00:39:45.724 INFO:teuthology.orchestra.run.smithi121.stdout:(5/13): python3-podman-4.9.0-1.module_el8+847+7 1.8 MB/s | 180 kB 00:00 2024-04-23T00:39:45.745 INFO:teuthology.orchestra.run.smithi153.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 248 kB/s | 25 kB 00:00 2024-04-23T00:39:45.824 INFO:teuthology.orchestra.run.smithi121.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 248 kB/s | 25 kB 00:00 2024-04-23T00:39:45.853 INFO:teuthology.orchestra.run.smithi153.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 869 kB/s | 94 kB 00:00 2024-04-23T00:39:45.891 INFO:teuthology.orchestra.run.smithi121.stdout:(7/13): netavark-1.10.3-1.module_el8+951+32019c 12 MB/s | 4.1 MB 00:00 2024-04-23T00:39:45.912 INFO:teuthology.orchestra.run.smithi153.stdout:(8/13): netavark-1.10.3-1.module_el8+951+32019c 9.1 MB/s | 4.1 MB 00:00 2024-04-23T00:39:45.925 INFO:teuthology.orchestra.run.smithi121.stdout:(8/13): python3-pyxdg-0.25-16.el8.noarch.rpm 935 kB/s | 94 kB 00:00 2024-04-23T00:39:46.012 INFO:teuthology.orchestra.run.smithi153.stdout:(9/13): buildah-1.33.6-2.module_el8+976+1136073 8.0 MB/s | 9.6 MB 00:01 2024-04-23T00:39:46.056 INFO:teuthology.orchestra.run.smithi121.stdout:(9/13): buildah-1.33.6-2.module_el8+976+1136073 8.1 MB/s | 9.6 MB 00:01 2024-04-23T00:39:46.112 INFO:teuthology.orchestra.run.smithi153.stdout:(10/13): udica-0.2.6-20.module_el8+804+f131391c 528 kB/s | 53 kB 00:00 2024-04-23T00:39:46.158 INFO:teuthology.orchestra.run.smithi121.stdout:(10/13): udica-0.2.6-20.module_el8+804+f131391c 516 kB/s | 53 kB 00:00 2024-04-23T00:39:46.171 INFO:teuthology.orchestra.run.smithi153.stdout:(11/13): toolbox-0.0.99.5-2.module_el8+971+3d3d 9.8 MB/s | 2.5 MB 00:00 2024-04-23T00:39:46.230 INFO:teuthology.orchestra.run.smithi153.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 350 kB/s | 41 kB 00:00 2024-04-23T00:39:46.251 INFO:teuthology.orchestra.run.smithi121.stdout:(11/13): toolbox-0.0.99.5-2.module_el8+971+3d3d 7.8 MB/s | 2.5 MB 00:00 2024-04-23T00:39:46.276 INFO:teuthology.orchestra.run.smithi121.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 350 kB/s | 41 kB 00:00 2024-04-23T00:39:46.418 INFO:teuthology.orchestra.run.smithi121.stdout:(13/13): skopeo-1.14.3-0.1.module_el8+885+7da14 17 MB/s | 8.8 MB 00:00 2024-04-23T00:39:46.418 INFO:teuthology.orchestra.run.smithi121.stdout:-------------------------------------------------------------------------------- 2024-04-23T00:39:46.418 INFO:teuthology.orchestra.run.smithi121.stdout:Total 16 MB/s | 27 MB 00:01 2024-04-23T00:39:46.421 INFO:teuthology.orchestra.run.smithi153.stdout:(13/13): skopeo-1.14.3-0.1.module_el8+885+7da14 16 MB/s | 8.8 MB 00:00 2024-04-23T00:39:46.422 INFO:teuthology.orchestra.run.smithi153.stdout:-------------------------------------------------------------------------------- 2024-04-23T00:39:46.422 INFO:teuthology.orchestra.run.smithi153.stdout:Total 16 MB/s | 27 MB 00:01 2024-04-23T00:39:46.690 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction check 2024-04-23T00:39:46.696 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction check 2024-04-23T00:39:46.711 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction check succeeded. 2024-04-23T00:39:46.712 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction test 2024-04-23T00:39:46.718 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction check succeeded. 2024-04-23T00:39:46.718 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction test 2024-04-23T00:39:46.968 INFO:teuthology.orchestra.run.smithi121.stdout:Transaction test succeeded. 2024-04-23T00:39:46.972 INFO:teuthology.orchestra.run.smithi121.stdout:Running transaction 2024-04-23T00:39:46.988 INFO:teuthology.orchestra.run.smithi153.stdout:Transaction test succeeded. 2024-04-23T00:39:46.991 INFO:teuthology.orchestra.run.smithi153.stdout:Running transaction 2024-04-23T00:39:47.580 INFO:teuthology.orchestra.run.smithi121.stdout: Preparing : 1/1 2024-04-23T00:39:47.642 INFO:teuthology.orchestra.run.smithi153.stdout: Preparing : 1/1 2024-04-23T00:39:48.561 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2024-04-23T00:39:48.595 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2024-04-23T00:39:48.776 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2/13 2024-04-23T00:39:48.788 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2/13 2024-04-23T00:39:48.942 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/13 2024-04-23T00:39:48.946 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/13 2024-04-23T00:39:49.134 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 4/13 2024-04-23T00:39:49.156 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 4/13 2024-04-23T00:39:49.374 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 5/13 2024-04-23T00:39:49.402 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 5/13 2024-04-23T00:39:49.927 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 6/13 2024-04-23T00:39:49.952 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 6/13 2024-04-23T00:39:50.893 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 7/13 2024-04-23T00:39:50.915 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 7/13 2024-04-23T00:39:51.146 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 8/13 2024-04-23T00:39:51.153 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 8/13 2024-04-23T00:39:51.359 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 9/13 2024-04-23T00:39:51.373 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 9/13 2024-04-23T00:39:51.790 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 10/13 2024-04-23T00:39:51.846 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 10/13 2024-04-23T00:39:52.013 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 11/13 2024-04-23T00:39:52.069 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 11/13 2024-04-23T00:39:52.212 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-04-23T00:39:52.275 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-04-23T00:39:52.433 INFO:teuthology.orchestra.run.smithi153.stdout: Installing : cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 13/13 2024-04-23T00:39:52.478 INFO:teuthology.orchestra.run.smithi121.stdout: Installing : cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 13/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Running scriptlet: cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 13/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 1/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 2/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 3/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 5/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 6/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 7/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2024-04-23T00:39:52.851 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 10/13 2024-04-23T00:39:52.852 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 11/13 2024-04-23T00:39:52.852 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Running scriptlet: cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 13/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x8 1/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 2/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : cockpit-podman-84.1-1.module_el8+977+8ab54275.noar 3/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 5/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 6/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-podman-4.9.0-1.module_el8+847+7863d4e6.noa 7/13 2024-04-23T00:39:52.880 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2024-04-23T00:39:52.881 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2024-04-23T00:39:52.881 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 10/13 2024-04-23T00:39:52.881 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 11/13 2024-04-23T00:39:52.881 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : udica-0.2.6-20.module_el8+804+f131391c.noarch 12/13 2024-04-23T00:39:53.342 INFO:teuthology.orchestra.run.smithi153.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2024-04-23T00:39:53.342 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:53.342 INFO:teuthology.orchestra.run.smithi153.stdout:Installed: 2024-04-23T00:39:53.342 INFO:teuthology.orchestra.run.smithi153.stdout: aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: cockpit-podman-84.1-1.module_el8+977+8ab54275.noarch 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: python3-podman-4.9.0-1.module_el8+847+7863d4e6.noarch 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: udica-0.2.6-20.module_el8+804+f131391c.noarch 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: yajl-2.1.0-12.el8.x86_64 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:39:53.343 INFO:teuthology.orchestra.run.smithi153.stdout:Complete! 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout:Installed: 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: aardvark-dns-2:1.10.0-1.module_el8+871+552be958.x86_64 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: buildah-2:1.33.6-2.module_el8+976+11360731.x86_64 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: cockpit-podman-84.1-1.module_el8+977+8ab54275.noarch 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: crun-1.14.3-2.module_el8+968+fbb249c7.x86_64 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: netavark-2:1.10.3-1.module_el8+951+32019cde.x86_64 2024-04-23T00:39:53.385 INFO:teuthology.orchestra.run.smithi121.stdout: python3-podman-4.9.0-1.module_el8+847+7863d4e6.noarch 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: skopeo-2:1.14.3-0.1.module_el8+885+7da147f3.x86_64 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: toolbox-0.0.99.5-2.module_el8+971+3d3df00d.x86_64 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: udica-0.2.6-20.module_el8+804+f131391c.noarch 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: yajl-2.1.0-12.el8.x86_64 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:39:53.386 INFO:teuthology.orchestra.run.smithi121.stdout:Complete! 2024-04-23T00:39:53.625 DEBUG:teuthology.parallel:result is None 2024-04-23T00:39:53.664 DEBUG:teuthology.parallel:result is None 2024-04-23T00:39:53.665 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-23T00:39:53.761 INFO:tasks.cephadm:Config: {'avoid_pacific_features': True, 'cephadm_branch': 'v16.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v16.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '646b8c7d293d8ec31a40c69f7b3237b19c1be24d'} 2024-04-23T00:39:53.761 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v16.2.0 2024-04-23T00:39:53.762 INFO:tasks.cephadm:Cluster fsid is 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:39:53.762 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-23T00:39:53.762 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-23T00:39:53.763 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi121': '172.21.15.121', 'mon.smithi153': '172.21.15.153'} 2024-04-23T00:39:53.763 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-23T00:39:53.763 DEBUG:teuthology.orchestra.run.smithi121:> sudo hostname $(hostname -s) 2024-04-23T00:39:53.791 DEBUG:teuthology.orchestra.run.smithi153:> sudo hostname $(hostname -s) 2024-04-23T00:39:53.819 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v16.2.0)... 2024-04-23T00:39:53.819 DEBUG:teuthology.orchestra.run.smithi121:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.100 INFO:teuthology.orchestra.run.smithi121.stdout:-rw-rw-r--. 1 ubuntu ubuntu 288386 Apr 23 00:39 /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.100 DEBUG:teuthology.orchestra.run.smithi153:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.363 INFO:teuthology.orchestra.run.smithi153.stdout:-rw-rw-r--. 1 ubuntu ubuntu 288386 Apr 23 00:39 /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.363 DEBUG:teuthology.orchestra.run.smithi121:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.383 DEBUG:teuthology.orchestra.run.smithi153:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-23T00:39:54.411 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v16.2.0 on all hosts... 2024-04-23T00:39:54.411 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 pull 2024-04-23T00:39:54.425 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 pull 2024-04-23T00:39:54.720 INFO:teuthology.orchestra.run.smithi153.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-04-23T00:39:54.721 INFO:teuthology.orchestra.run.smithi121.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-04-23T00:40:12.556 INFO:teuthology.orchestra.run.smithi153.stdout:{ 2024-04-23T00:40:12.556 INFO:teuthology.orchestra.run.smithi153.stdout: "ceph_version": "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)", 2024-04-23T00:40:12.556 INFO:teuthology.orchestra.run.smithi153.stdout: "image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", 2024-04-23T00:40:12.556 INFO:teuthology.orchestra.run.smithi153.stdout: "repo_digests": [ 2024-04-23T00:40:12.556 INFO:teuthology.orchestra.run.smithi153.stdout: "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", 2024-04-23T00:40:12.557 INFO:teuthology.orchestra.run.smithi153.stdout: "quay.io/ceph/ceph@sha256:9b04c0f15704c49591640a37c7adfd40ffad0a4b42fecb950c3407687cb4f29a" 2024-04-23T00:40:12.557 INFO:teuthology.orchestra.run.smithi153.stdout: ] 2024-04-23T00:40:12.557 INFO:teuthology.orchestra.run.smithi153.stdout:} 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph_version": "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)", 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: "image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: "repo_digests": [ 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: "quay.io/ceph/ceph@sha256:9b04c0f15704c49591640a37c7adfd40ffad0a4b42fecb950c3407687cb4f29a" 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2024-04-23T00:40:13.051 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:40:13.069 DEBUG:teuthology.orchestra.run.smithi121:> sudo mkdir -p /etc/ceph 2024-04-23T00:40:13.099 DEBUG:teuthology.orchestra.run.smithi153:> sudo mkdir -p /etc/ceph 2024-04-23T00:40:13.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 777 /etc/ceph 2024-04-23T00:40:13.166 DEBUG:teuthology.orchestra.run.smithi153:> sudo chmod 777 /etc/ceph 2024-04-23T00:40:13.200 INFO:tasks.cephadm:Writing seed config... 2024-04-23T00:40:13.201 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-23T00:40:13.202 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-23T00:40:13.203 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:40:13.203 DEBUG:teuthology.orchestra.run.smithi121:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-23T00:40:13.224 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 004558ac-010a-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-23T00:40:13.225 DEBUG:teuthology.orchestra.run.smithi121:mon.smithi121> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service 2024-04-23T00:40:13.267 INFO:tasks.cephadm:Bootstrapping... 2024-04-23T00:40:13.267 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 -v bootstrap --fsid 004558ac-010a-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.121 && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-23T00:40:13.293 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:-- Logs begin at Tue 2024-04-23 00:33:19 UTC. -- 2024-04-23T00:40:13.453 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman --version 2024-04-23T00:40:13.473 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: podman version 4.9.4-dev 2024-04-23T00:40:13.474 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying podman|docker is present... 2024-04-23T00:40:13.474 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman --version 2024-04-23T00:40:13.492 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: podman version 4.9.4-dev 2024-04-23T00:40:13.493 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying lvm2 is present... 2024-04-23T00:40:13.494 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying time synchronization is in place... 2024-04-23T00:40:13.494 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled chrony.service 2024-04-23T00:40:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-23T00:40:13.500 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active chrony.service 2024-04-23T00:40:13.505 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:40:13.506 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled chronyd.service 2024-04-23T00:40:13.511 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: enabled 2024-04-23T00:40:13.512 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active chronyd.service 2024-04-23T00:40:13.517 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: active 2024-04-23T00:40:13.517 INFO:teuthology.orchestra.run.smithi121.stderr:Unit chronyd.service is enabled and running 2024-04-23T00:40:13.517 INFO:teuthology.orchestra.run.smithi121.stderr:Repeating the final host check... 2024-04-23T00:40:13.517 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman --version 2024-04-23T00:40:13.535 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: podman version 4.9.4-dev 2024-04-23T00:40:13.536 INFO:teuthology.orchestra.run.smithi121.stderr:podman|docker (/bin/podman) is present 2024-04-23T00:40:13.537 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl is present 2024-04-23T00:40:13.537 INFO:teuthology.orchestra.run.smithi121.stderr:lvcreate is present 2024-04-23T00:40:13.537 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled chrony.service 2024-04-23T00:40:13.542 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-23T00:40:13.542 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active chrony.service 2024-04-23T00:40:13.547 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:40:13.548 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled chronyd.service 2024-04-23T00:40:13.553 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: enabled 2024-04-23T00:40:13.554 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active chronyd.service 2024-04-23T00:40:13.559 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: active 2024-04-23T00:40:13.559 INFO:teuthology.orchestra.run.smithi121.stderr:Unit chronyd.service is enabled and running 2024-04-23T00:40:13.560 INFO:teuthology.orchestra.run.smithi121.stderr:Host looks OK 2024-04-23T00:40:13.560 INFO:teuthology.orchestra.run.smithi121.stderr:Cluster fsid: 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:13.560 INFO:teuthology.orchestra.run.smithi121.stderr:Acquiring lock 140438534033536 on /run/cephadm/004558ac-010a-11ef-bc93-c7b262605968.lock 2024-04-23T00:40:13.560 INFO:teuthology.orchestra.run.smithi121.stderr:Lock 140438534033536 acquired on /run/cephadm/004558ac-010a-11ef-bc93-c7b262605968.lock 2024-04-23T00:40:13.561 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying IP 172.21.15.121 port 3300 ... 2024-04-23T00:40:13.561 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying IP 172.21.15.121 port 6789 ... 2024-04-23T00:40:13.561 INFO:teuthology.orchestra.run.smithi121.stderr:Base mon IP is 172.21.15.121, final addrv is [v2:172.21.15.121:3300,v1:172.21.15.121:6789] 2024-04-23T00:40:13.561 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /sbin/ip route ls 2024-04-23T00:40:13.563 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.121 metric 100 2024-04-23T00:40:13.563 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.121 metric 100 2024-04-23T00:40:13.563 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /sbin/ip -6 route ls 2024-04-23T00:40:13.565 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-23T00:40:13.565 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-23T00:40:13.566 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2024-04-23T00:40:13.566 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /sbin/ip -6 addr ls 2024-04-23T00:40:13.568 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-23T00:40:13.568 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-23T00:40:13.568 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-23T00:40:13.568 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-23T00:40:13.568 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:cba3/64 scope link 2024-04-23T00:40:13.569 INFO:teuthology.orchestra.run.smithi121.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-23T00:40:13.569 INFO:teuthology.orchestra.run.smithi121.stderr:Mon IP 172.21.15.121 is in CIDR network 172.21.0.0/20 2024-04-23T00:40:13.569 INFO:teuthology.orchestra.run.smithi121.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-23T00:40:13.570 INFO:teuthology.orchestra.run.smithi121.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-04-23T00:40:13.570 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman pull quay.io/ceph/ceph:v16.2.0 2024-04-23T00:40:13.647 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v16.2.0... 2024-04-23T00:40:14.021 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Getting image source signatures 2024-04-23T00:40:14.021 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Copying blob sha256:958e3c20ad47ec7f3ec0446a4a630f2285dd6b0a4c969f4f702f7c2e78f3d68b 2024-04-23T00:40:14.021 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Copying blob sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621 2024-04-23T00:40:14.021 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Copying config sha256:24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 2024-04-23T00:40:14.168 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: Writing manifest to image destination 2024-04-23T00:40:14.320 INFO:teuthology.orchestra.run.smithi121.stderr:/bin/podman: 24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 2024-04-23T00:40:14.330 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --version 2024-04-23T00:40:14.832 INFO:teuthology.orchestra.run.smithi121.stderr:ceph: ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable) 2024-04-23T00:40:15.459 INFO:teuthology.orchestra.run.smithi121.stderr:Ceph version: ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable) 2024-04-23T00:40:15.460 INFO:teuthology.orchestra.run.smithi121.stderr:Extracting ceph user uid/gid from container image... 2024-04-23T00:40:15.460 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 -c %u %g /var/lib/ceph 2024-04-23T00:40:16.037 INFO:teuthology.orchestra.run.smithi121.stderr:stat: 167 167 2024-04-23T00:40:16.665 INFO:teuthology.orchestra.run.smithi121.stderr:Creating initial keys... 2024-04-23T00:40:16.665 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-04-23T00:40:17.168 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-authtool: AQBxAydmlrbVBBAA0dkIGy/LCQPxANGsgwnCwg== 2024-04-23T00:40:17.626 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-04-23T00:40:18.157 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-authtool: AQByAydmPXwpAhAAF1DGMRg06nI0G3qQBZu15A== 2024-04-23T00:40:18.618 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-04-23T00:40:19.138 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-authtool: AQBzAydmuqyKAxAAV+SjALsyLbto5zmJVmRGlA== 2024-04-23T00:40:19.727 INFO:teuthology.orchestra.run.smithi121.stderr:Creating initial monmap... 2024-04-23T00:40:19.728 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/monmaptool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /tmp/ceph-tmpip3znm95:/tmp/monmap:z quay.io/ceph/ceph:v16.2.0 --create --clobber --fsid 004558ac-010a-11ef-bc93-c7b262605968 --addv smithi121 [v2:172.21.15.121:3300,v1:172.21.15.121:6789] /tmp/monmap 2024-04-23T00:40:20.236 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-23T00:40:20.236 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:20.236 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-23T00:40:20.836 INFO:teuthology.orchestra.run.smithi121.stderr:monmaptool for smithi121 [v2:172.21.15.121:3300,v1:172.21.15.121:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-23T00:40:20.836 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/monmaptool: set fsid to 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:20.836 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-23T00:40:20.836 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:40:20.836 INFO:teuthology.orchestra.run.smithi121.stderr:Creating mon... 2024-04-23T00:40:20.838 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-mon --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121:/var/lib/ceph/mon/ceph-smithi121:z -v /tmp/ceph-tmpowiaw8x9:/tmp/keyring:z -v /tmp/ceph-tmpip3znm95:/tmp/monmap:z quay.io/ceph/ceph:v16.2.0 --mkfs -i smithi121 --fsid 004558ac-010a-11ef-bc93-c7b262605968 -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.271+0000 7fefa6cda700 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.271+0000 7fefa6cda700 1 imported monmap: 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: epoch 0 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: last_changed 2024-04-23T00:40:20.161921+0000 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: created 2024-04-23T00:40:20.161921+0000 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: election_strategy: 1 2024-04-23T00:40:21.374 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon.smithi121 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.271+0000 7fefa6cda700 0 /usr/bin/ceph-mon: set fsid to 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: RocksDB version: 6.8.1 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Compile date Mar 30 2021 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: DB SUMMARY 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi121/store.db dir, Total Num: 0, files: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi121/store.db: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.375 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.error_if_exists: 0 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.create_if_missing: 1 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.paranoid_checks: 1 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.env: 0x562512aaa1c0 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.fs: Posix File System 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.info_log: 0x562514904c60 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.statistics: (nil) 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.use_fsync: 0 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_log_file_size: 0 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-23T00:40:21.376 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.allow_fallocate: 1 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.use_direct_reads: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.create_missing_column_families: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.db_log_dir: 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi121/store.db 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_subcompactions: 1 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_background_flushes: -1 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-23T00:40:21.377 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.advise_random_on_open: 1 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.write_buffer_manager: 0x56251490f440 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.rate_limiter: (nil) 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-23T00:40:21.378 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.unordered_write: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.row_cache: None 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.wal_filter: None 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.preserve_deletes: 0 2024-04-23T00:40:21.379 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.two_write_queues: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.manual_wal_flush: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.atomic_flush: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.log_readahead_size: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_background_jobs: 2 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_background_compactions: -1 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-23T00:40:21.380 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_total_wal_size: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.max_open_files: -1 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.bytes_per_sync: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Compression algorithms supported: 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kZSTD supported: 0 2024-04-23T00:40:21.381 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kXpressCompression supported: 0 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kLZ4Compression supported: 1 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kBZip2Compression supported: 0 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kZlibCompression supported: 1 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: kSnappyCompression supported: 1 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.272+0000 7fefa6cda700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000001 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.382 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.merge_operator: 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_filter: None 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_filter_factory: None 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56251481e0d0) 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: index_type: 0 2024-04-23T00:40:21.383 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: index_shortening: 1 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: checksum: 1 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_cache: 0x562514854d10 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_cache_options: 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: capacity : 536870912 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2024-04-23T00:40:21.384 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_size: 4096 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: partition_filters: 0 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2024-04-23T00:40:21.385 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: verify_compression: 0 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: format_version: 2 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: block_align: 0 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression: NoCompression 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.num_levels: 7 2024-04-23T00:40:21.386 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.level: 32767 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-23T00:40:21.387 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compression_opts.enabled: false 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-23T00:40:21.388 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.arena_block_size: 4194304 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-23T00:40:21.389 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.table_properties_collectors: 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.inplace_update_support: 0 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-23T00:40:21.390 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.bloom_locality: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.max_successive_merges: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.force_consistency_checks: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.ttl: 2592000 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.273+0000 7fefa6cda700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2024-04-23T00:40:21.391 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.276+0000 7fefa6cda700 4 rocksdb: DB pointer 0x562514919800 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.276+0000 7fef8fb75700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.276+0000 7fef8fb75700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-23T00:40:21.392 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-04-23T00:40:21.393 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.394 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-04-23T00:40:21.395 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-04-23T00:40:21.396 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: 2024-04-23T00:40:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.277+0000 7fefa6cda700 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2024-04-23T00:40:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.278+0000 7fefa6cda700 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2024-04-23T00:40:21.397 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph-mon: debug 2024-04-23T00:40:21.278+0000 7fefa6cda700 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-smithi121 for mon.smithi121 2024-04-23T00:40:21.959 INFO:teuthology.orchestra.run.smithi121.stderr:create mon.smithi121 on 2024-04-23T00:40:21.959 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:21.964 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl disable ceph.target 2024-04-23T00:40:22.054 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl enable ceph.target 2024-04-23T00:40:22.060 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-23T00:40:22.138 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl start ceph.target 2024-04-23T00:40:22.144 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl enable ceph-004558ac-010a-11ef-bc93-c7b262605968.target 2024-04-23T00:40:22.151 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-004558ac-010a-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968.target. 2024-04-23T00:40:22.151 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-004558ac-010a-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968.target. 2024-04-23T00:40:22.231 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl start ceph-004558ac-010a-11ef-bc93-c7b262605968.target 2024-04-23T00:40:22.255 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl daemon-reload 2024-04-23T00:40:22.338 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T00:40:22.368 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl reset-failed ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T00:40:22.374 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Failed to reset failed state of unit ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service: Unit ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service not loaded. 2024-04-23T00:40:22.374 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl enable ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T00:40:22.380 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Created symlink /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968.target.wants/ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service → /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968@.service. 2024-04-23T00:40:22.461 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl start ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T00:40:22.500 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:22 smithi121 systemd[1]: Starting Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T00:40:22.969 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled firewalld.service 2024-04-23T00:40:22.978 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: disabled 2024-04-23T00:40:22.979 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active firewalld.service 2024-04-23T00:40:22.985 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:40:22.986 INFO:teuthology.orchestra.run.smithi121.stderr:firewalld.service is not enabled 2024-04-23T00:40:22.986 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to enable service . firewalld.service is not available 2024-04-23T00:40:22.986 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to open ports <[]>. firewalld.service is not available 2024-04-23T00:40:22.987 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mon to start... 2024-04-23T00:40:22.987 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mon... 2024-04-23T00:40:22.987 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121:/var/lib/ceph/mon/ceph-smithi121:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status 2024-04-23T00:40:23.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.936299+0000 mon.smithi121 (mon.0) 0 : [INF] mkfs 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:23.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.926790+0000 mon.smithi121 (mon.0) 1 : cluster [INF] mon.smithi121 is new leader, mons smithi121 in quorum (ranks 0) 2024-04-23T00:40:23.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:22 smithi121 bash[34970]: 8c38d6f54685b6f9b5a7283f2d16b2b546d200e7d8d092703494f7549954338c 2024-04-23T00:40:23.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:22 smithi121 systemd[1]: Started Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: cluster: 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: id: 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: services: 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi121 (age 0.657863s) 2024-04-23T00:40:23.584 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: data: 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: pgs: 2024-04-23T00:40:23.585 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:24.174 INFO:teuthology.orchestra.run.smithi121.stderr:mon is available 2024-04-23T00:40:24.174 INFO:teuthology.orchestra.run.smithi121.stderr:Assimilating anything we can from ceph.conf... 2024-04-23T00:40:24.175 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z -v /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121:/var/lib/ceph/mon/ceph-smithi121:z quay.io/ceph/ceph:v16.2.0 config assimilate-conf -i /var/lib/ceph/mon/ceph-smithi121/config 2024-04-23T00:40:24.177 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.934855+0000 mon.smithi121 (mon.0) 2 : cluster [INF] mon.smithi121 is new leader, mons smithi121 in quorum (ranks 0) 2024-04-23T00:40:24.177 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.935018+0000 mon.smithi121 (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0]} 2024-04-23T00:40:24.177 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.938173+0000 mon.smithi121 (mon.0) 4 : cluster [DBG] fsmap 2024-04-23T00:40:24.177 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.942272+0000 mon.smithi121 (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-04-23T00:40:24.177 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: cluster 2024-04-23T00:40:22.942644+0000 mon.smithi121 (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-04-23T00:40:24.178 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: audit 2024-04-23T00:40:23.592690+0000 mon.smithi121 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.121:0/2522579725' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-23T00:40:24.716 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: [global] 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: fsid = 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: log_to_journald = false 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.121:3300,v1:172.21.15.121:6789] 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: [mgr] 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:24.717 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: [mon] 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_warn_on_insecure_global_id_reclaim = false 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: [osd] 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: osd_memory_target_autotune = true 2024-04-23T00:40:24.718 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-23T00:40:25.036 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: audit 2024-04-23T00:40:24.721199+0000 mon.smithi121 (mon.0) 8 : audit [INF] from='client.? 172.21.15.121:0/2315435316' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-23T00:40:25.036 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: audit 2024-04-23T00:40:24.723655+0000 mon.smithi121 (mon.0) 9 : audit [INF] from='client.? 172.21.15.121:0/2315435316' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-23T00:40:25.289 INFO:teuthology.orchestra.run.smithi121.stderr:Generating new minimal ceph.conf... 2024-04-23T00:40:25.289 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z -v /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121:/var/lib/ceph/mon/ceph-smithi121:z quay.io/ceph/ceph:v16.2.0 config generate-minimal-conf -o /var/lib/ceph/mon/ceph-smithi121/config 2024-04-23T00:40:26.150 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: audit 2024-04-23T00:40:25.874912+0000 mon.smithi121 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.121:0/3011375263' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:40:26.379 INFO:teuthology.orchestra.run.smithi121.stderr:Restarting the monitor... 2024-04-23T00:40:26.379 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl restart ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T00:40:26.476 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:26 smithi121 systemd[1]: Stopping Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T00:40:26.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: debug 2024-04-23T00:40:26.483+0000 7f36c442e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi121 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-23T00:40:26.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[34985]: debug 2024-04-23T00:40:26.483+0000 7f36c442e700 -1 mon.smithi121@0(leader) e1 *** Got Signal Terminated *** 2024-04-23T00:40:27.231 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:26 smithi121 podman[35307]: ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121 2024-04-23T00:40:27.616 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service: Succeeded. 2024-04-23T00:40:27.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 systemd[1]: Stopped Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T00:40:27.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 systemd[1]: Starting Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T00:40:28.035 INFO:teuthology.orchestra.run.smithi121.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-23T00:40:28.036 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mon public_network 172.21.0.0/20 2024-04-23T00:40:28.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.987+0000 7ff4bf5d8700 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-23T00:40:28.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.987+0000 7ff4bf5d8700 0 ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable), process ceph-mon, pid 7 2024-04-23T00:40:28.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.987+0000 7ff4bf5d8700 0 pidfile_write: ignore empty --pid-file 2024-04-23T00:40:28.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.991+0000 7ff4bf5d8700 0 load: jerasure load: lrc load: isa 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: RocksDB version: 6.8.1 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Compile date Mar 30 2021 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: DB SUMMARY 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: CURRENT file: CURRENT 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: IDENTITY file: IDENTITY 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi121/store.db dir, Total Num: 1, files: 000004.sst 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi121/store.db: 000006.log size: 83442 ; 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.error_if_exists: 0 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.create_if_missing: 0 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.paranoid_checks: 1 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.env: 0x557c59e151c0 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.fs: Posix File System 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.info_log: 0x557c5c6bf6e0 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.statistics: (nil) 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.use_fsync: 0 2024-04-23T00:40:28.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_log_file_size: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.allow_fallocate: 1 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.use_direct_reads: 0 2024-04-23T00:40:28.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.create_missing_column_families: 0 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.db_log_dir: 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi121/store.db 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_subcompactions: 1 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_background_flushes: -1 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.advise_random_on_open: 1 2024-04-23T00:40:28.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.write_buffer_manager: 0x557c5c6c9ec0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.rate_limiter: (nil) 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-23T00:40:28.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.unordered_write: 0 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.row_cache: None 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.wal_filter: None 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-23T00:40:28.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.preserve_deletes: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.two_write_queues: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.manual_wal_flush: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.atomic_flush: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.log_readahead_size: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_background_jobs: 2 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_background_compactions: -1 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-23T00:40:28.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_total_wal_size: 0 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_open_files: -1 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bytes_per_sync: 0 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Compression algorithms supported: 2024-04-23T00:40:28.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kZSTD supported: 0 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kXpressCompression supported: 0 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kLZ4Compression supported: 1 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kBZip2Compression supported: 0 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kZlibCompression supported: 1 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: kSnappyCompression supported: 1 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000005 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-04-23T00:40:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.merge_operator: 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_filter: None 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_filter_factory: None 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557c5c5d80d8) 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cache_index_and_filter_blocks: 1 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: pin_top_level_index_and_filter: 1 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: index_type: 0 2024-04-23T00:40:28.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: data_block_index_type: 0 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: index_shortening: 1 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: data_block_hash_table_util_ratio: 0.750000 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: hash_index_allow_collision: 1 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: checksum: 1 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: no_block_cache: 0 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_cache: 0x557c5c60ef10 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_cache_name: BinnedLRUCache 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_cache_options: 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: capacity : 536870912 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: num_shard_bits : 4 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: strict_capacity_limit : 0 2024-04-23T00:40:28.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: high_pri_pool_ratio: 0.000 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_cache_compressed: (nil) 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: persistent_cache: (nil) 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_size: 4096 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_size_deviation: 10 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_restart_interval: 16 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: index_block_restart_interval: 1 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: metadata_block_size: 4096 2024-04-23T00:40:28.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: partition_filters: 0 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: use_delta_encoding: 1 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: whole_key_filtering: 1 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: verify_compression: 0 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: read_amp_bytes_per_bit: 0 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: format_version: 2 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: enable_index_compression: 1 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: block_align: 0 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-23T00:40:28.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression: NoCompression 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.num_levels: 7 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-23T00:40:28.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.level: 32767 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compression_opts.enabled: false 2024-04-23T00:40:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-23T00:40:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.arena_block_size: 4194304 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-23T00:40:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.table_properties_collectors: 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.inplace_update_support: 0 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-23T00:40:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.bloom_locality: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.max_successive_merges: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.force_consistency_checks: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.ttl: 2592000 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.992+0000 7ff4bf5d8700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.993+0000 7ff4bf5d8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1713832827994234, "job": 1, "event": "recovery_started", "log_files": [6]} 2024-04-23T00:40:28.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.993+0000 7ff4bf5d8700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.993+0000 7ff4bf5d8700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.993+0000 7ff4bf5d8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1713832827995047, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 80511, "table_properties": {"data_size": 78770, "index_size": 346, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 8836, "raw_average_key_size": 46, "raw_value_size": 73560, "raw_average_value_size": 383, "num_data_blocks": 11, "num_entries": 192, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713832827, "oldest_key_time": 3, "file_creation_time": 0}} 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.993+0000 7ff4bf5d8700 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.994+0000 7ff4bf5d8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1713832827995422, "job": 1, "event": "recovery_finished"} 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.994+0000 7ff4bf5d8700 4 rocksdb: DB pointer 0x557c5c6d3800 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.995+0000 7ff4a63f2700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.995+0000 7ff4a63f2700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** DB Stats ** 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T00:40:28.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** Compaction Stats [default] ** 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: L0 2/0 80.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Sum 2/0 80.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** Compaction Stats [default] ** 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:28.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Flush(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(Keys): cumulative 0, interval 0 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Cumulative compaction: 0.00 GB write, 29.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Interval compaction: 0.00 GB write, 29.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** File Read Latency Histogram By Level [default] ** 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** Compaction Stats [default] ** 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: L0 2/0 80.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Sum 2/0 80.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** Compaction Stats [default] ** 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.1 0.00 0.00 1 0.001 0 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Flush(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: AddFile(Keys): cumulative 0, interval 0 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Cumulative compaction: 0.00 GB write, 28.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T00:40:28.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: ** File Read Latency Histogram By Level [default] ** 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.995+0000 7ff4bf5d8700 0 starting mon.smithi121 rank 0 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi121 fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 1 mon.smithi121@-1(???) e1 preinit fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).mds e1 new map 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).mds e1 print_map 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: e1 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: legacy client fscid: -1 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: No filesystems configured 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-23T00:40:28.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 0 mon.smithi121@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:25.874912+0000 mon.smithi121 (mon.0) 10 : audit [DBG] from='client.? 172.21.15.121:0/3011375263' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:40:27.996+0000 7ff4bf5d8700 1 mon.smithi121@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 bash[35378]: ee62d2b36a2d97a576349a906e0ca50fa8d9cf9d0ae0dd920d445bbb31a6d60b 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 systemd[1]: Started Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:28.012936+0000 mon.smithi121 (mon.0) 1 : cluster [INF] mon.smithi121 is new leader, mons smithi121 in quorum (ranks 0) 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:28.012996+0000 mon.smithi121 (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0]} 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:28.013025+0000 mon.smithi121 (mon.0) 3 : cluster [DBG] fsmap 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:28.013054+0000 mon.smithi121 (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-04-23T00:40:28.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:28.013395+0000 mon.smithi121 (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-04-23T00:40:29.175 INFO:teuthology.orchestra.run.smithi121.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-23T00:40:29.176 INFO:teuthology.orchestra.run.smithi121.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-23T00:40:29.176 INFO:teuthology.orchestra.run.smithi121.stderr:Creating mgr... 2024-04-23T00:40:29.176 INFO:teuthology.orchestra.run.smithi121.stderr:Verifying port 9283 ... 2024-04-23T00:40:29.178 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl daemon-reload 2024-04-23T00:40:29.278 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso 2024-04-23T00:40:29.300 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl reset-failed ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso 2024-04-23T00:40:29.306 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Failed to reset failed state of unit ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso.service: Unit ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso.service not loaded. 2024-04-23T00:40:29.306 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl enable ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso 2024-04-23T00:40:29.312 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: Created symlink /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968.target.wants/ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso.service → /etc/systemd/system/ceph-004558ac-010a-11ef-bc93-c7b262605968@.service. 2024-04-23T00:40:29.397 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl start ceph-004558ac-010a-11ef-bc93-c7b262605968@mgr.smithi121.otezso 2024-04-23T00:40:29.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:28.609147+0000 mon.smithi121 (mon.0) 6 : audit [INF] from='client.? 172.21.15.121:0/2495061984' entity='client.admin' 2024-04-23T00:40:29.880 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled firewalld.service 2024-04-23T00:40:29.888 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: disabled 2024-04-23T00:40:29.888 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active firewalld.service 2024-04-23T00:40:29.895 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:40:29.896 INFO:teuthology.orchestra.run.smithi121.stderr:firewalld.service is not enabled 2024-04-23T00:40:29.896 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to enable service . firewalld.service is not available 2024-04-23T00:40:29.896 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to open ports <[]>. firewalld.service is not available 2024-04-23T00:40:29.896 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled firewalld.service 2024-04-23T00:40:29.903 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: disabled 2024-04-23T00:40:29.904 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active firewalld.service 2024-04-23T00:40:29.911 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:40:29.912 INFO:teuthology.orchestra.run.smithi121.stderr:firewalld.service is not enabled 2024-04-23T00:40:29.912 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-23T00:40:29.912 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mgr to start... 2024-04-23T00:40:29.912 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mgr... 2024-04-23T00:40:29.912 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-04-23T00:40:30.535 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsid": "004558ac-010a-11ef-bc93-c7b262605968", 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "health": { 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "checks": {}, 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mutes": [] 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum": [ 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 0 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "smithi121" 2024-04-23T00:40:30.536 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:30.537 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-23T00:40:30.537 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "monmap": { 2024-04-23T00:40:30.537 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:30.537 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osdmap": { 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-23T00:40:30.538 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgmap": { 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.539 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsmap": { 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "available": false, 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modules": [ 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "iostat", 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "restful" 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:30.540 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "servicemap": { 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modified": "2024-04-23T00:40:22.936649+0000", 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-23T00:40:30.541 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:40:30.807 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:30.540165+0000 mon.smithi121 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.121:0/2363085471' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-23T00:40:31.069 INFO:teuthology.orchestra.run.smithi121.stderr:mgr not available, waiting (1/15)... 2024-04-23T00:40:33.072 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:34.661349+0000 mon.smithi121 (mon.0) 8 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:34.663960+0000 mon.smithi121 (mon.0) 9 : cluster [DBG] mgrmap e2: smithi121.otezso(active, starting, since 0.00271115s) 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.665042+0000 mon.smithi121 (mon.0) 10 : audit [DBG] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.665266+0000 mon.smithi121 (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.665467+0000 mon.smithi121 (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.665671+0000 mon.smithi121 (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.666145+0000 mon.smithi121 (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:34.679696+0000 mon.smithi121 (mon.0) 15 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.690088+0000 mon.smithi121 (mon.0) 16 : audit [INF] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.691463+0000 mon.smithi121 (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.692586+0000 mon.smithi121 (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.699587+0000 mon.smithi121 (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' 2024-04-23T00:40:34.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:34.704639+0000 mon.smithi121 (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.121:0/4156689948' entity='mgr.smithi121.otezso' 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsid": "004558ac-010a-11ef-bc93-c7b262605968", 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "health": { 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "checks": {}, 2024-04-23T00:40:35.313 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mutes": [] 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum": [ 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 0 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "smithi121" 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "monmap": { 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-23T00:40:35.314 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osdmap": { 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgmap": { 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-23T00:40:35.315 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsmap": { 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "available": false, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modules": [ 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "iostat", 2024-04-23T00:40:35.316 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "restful" 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "servicemap": { 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modified": "2024-04-23T00:40:22.936649+0000", 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-23T00:40:35.317 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:40:35.837 INFO:teuthology.orchestra.run.smithi121.stderr:mgr not available, waiting (2/15)... 2024-04-23T00:40:35.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:35.317373+0000 mon.smithi121 (mon.0) 21 : audit [DBG] from='client.? 172.21.15.121:0/3194562352' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-23T00:40:35.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:35.666304+0000 mon.smithi121 (mon.0) 22 : cluster [DBG] mgrmap e3: smithi121.otezso(active, since 1.00506s) 2024-04-23T00:40:37.840 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-04-23T00:40:38.046 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:36.721300+0000 mon.smithi121 (mon.0) 23 : cluster [DBG] mgrmap e4: smithi121.otezso(active, since 2s) 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsid": "004558ac-010a-11ef-bc93-c7b262605968", 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "health": { 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "checks": {}, 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mutes": [] 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.548 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum": [ 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 0 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "smithi121" 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "monmap": { 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osdmap": { 2024-04-23T00:40:38.549 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgmap": { 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-23T00:40:38.550 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "fsmap": { 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.551 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "available": true, 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modules": [ 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "iostat", 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "restful" 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ], 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "servicemap": { 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "modified": "2024-04-23T00:40:22.936649+0000", 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "services": {} 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: }, 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-23T00:40:38.552 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:40:39.102 INFO:teuthology.orchestra.run.smithi121.stderr:mgr is available 2024-04-23T00:40:39.102 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config-key set mgr/cephadm/ssh_user root 2024-04-23T00:40:39.283 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:38.552067+0000 mon.smithi121 (mon.0) 24 : audit [DBG] from='client.? 172.21.15.121:0/2616896276' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-23T00:40:39.809 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: set mgr/cephadm/ssh_user 2024-04-23T00:40:41.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:39.817119+0000 mon.smithi121 (mon.0) 25 : audit [INF] from='client.? 172.21.15.121:0/4291763839' entity='client.admin' 2024-04-23T00:40:42.503 INFO:teuthology.orchestra.run.smithi121.stderr:Enabling cephadm module... 2024-04-23T00:40:42.504 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr module enable cephadm 2024-04-23T00:40:46.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:45.870291+0000 mon.smithi121 (mon.0) 26 : audit [INF] from='client.? 172.21.15.121:0/129897191' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-23T00:40:47.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:45.923159+0000 mon.smithi121 (mon.0) 27 : audit [INF] from='client.? 172.21.15.121:0/129897191' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-23T00:40:47.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:45.923203+0000 mon.smithi121 (mon.0) 28 : cluster [DBG] mgrmap e5: smithi121.otezso(active, since 11s) 2024-04-23T00:40:48.381 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr stat 2024-04-23T00:40:49.124 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:40:49.124 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-23T00:40:49.124 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "available": true, 2024-04-23T00:40:49.124 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "active_name": "smithi121.otezso", 2024-04-23T00:40:49.125 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-23T00:40:49.125 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:40:49.468 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:49.133602+0000 mon.smithi121 (mon.0) 29 : audit [DBG] from='client.? 172.21.15.121:0/1703343451' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-23T00:40:49.738 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for the mgr to restart... 2024-04-23T00:40:49.738 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mgr epoch 5... 2024-04-23T00:40:49.738 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 tell mgr mgr_status 2024-04-23T00:40:51.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:50.776961+0000 mon.smithi121 (mon.0) 30 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T00:40:51.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:50.777378+0000 mon.smithi121 (mon.0) 31 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:40:51.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:50.781190+0000 mon.smithi121 (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-04-23T00:40:51.837 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:40:51.837 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-23T00:40:51.837 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "initialized": true 2024-04-23T00:40:51.837 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:40:52.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:50.832037+0000 mon.smithi121 (mon.0) 33 : cluster [DBG] mgrmap e6: smithi121.otezso(active, starting, since 0.0547384s) 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.834416+0000 mon.smithi121 (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.835036+0000 mon.smithi121 (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.835681+0000 mon.smithi121 (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.835813+0000 mon.smithi121 (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.835936+0000 mon.smithi121 (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:50.849668+0000 mon.smithi121 (mon.0) 39 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.862208+0000 mon.smithi121 (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.864788+0000 mon.smithi121 (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.867180+0000 mon.smithi121 (mon.0) 42 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.954812+0000 mon.smithi121 (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.955524+0000 mon.smithi121 (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:40:52.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.957706+0000 mon.smithi121 (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:40:52.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.959162+0000 mon.smithi121 (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:40:52.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.969055+0000 mon.smithi121 (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:40:52.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:50.970155+0000 mon.smithi121 (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:40:52.337 INFO:teuthology.orchestra.run.smithi121.stderr:mgr epoch 5 is available 2024-04-23T00:40:52.337 INFO:teuthology.orchestra.run.smithi121.stderr:Setting orchestrator backend to cephadm... 2024-04-23T00:40:52.337 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch set backend cephadm 2024-04-23T00:40:53.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:51.839100+0000 mgr.smithi121.otezso (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-23T00:40:53.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:51.839967+0000 mon.smithi121 (mon.0) 49 : cluster [DBG] mgrmap e7: smithi121.otezso(active, since 1.06266s) 2024-04-23T00:40:53.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:51.846245+0000 mgr.smithi121.otezso (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-23T00:40:53.608 INFO:teuthology.orchestra.run.smithi121.stderr:Generating ssh key... 2024-04-23T00:40:53.609 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 cephadm generate-key 2024-04-23T00:40:54.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:40:52.841473+0000 mon.smithi121 (mon.0) 50 : cluster [DBG] mgrmap e8: smithi121.otezso(active, since 2s) 2024-04-23T00:40:54.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:53.048368+0000 mon.smithi121 (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:54.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:53.051574+0000 mon.smithi121 (mon.0) 52 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:40:54.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:53.044380+0000 mgr.smithi121.otezso (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:40:54.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:54.625916+0000 mon.smithi121 (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:54.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:54.628435+0000 mon.smithi121 (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:55.149 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 cephadm get-pub-key 2024-04-23T00:40:55.805 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCa5zrw39+PxwD4/TOriluqagfVHileNnsYK+1H/Q3yzI+tEdcDo61nEAmsu/GN2UewTNDewcF0l0CMcoTw3OPq3wFqQLYhFC/BneIVJqvDsbeEbq2doL7ZIELZYGHZoMaVpbkCTUBsXhjBne75Kgdif0FFuEofSbsVbgHYChDF2mjOJvmizEpVjgGUGGs/hsmL2pg1+8Z8SwypVpg5fHjW8mYnITuBC5KIfK/wHJvSynby07nGWjeZUd/S/ve62tNlMpo9Nap5iywwbm7CAi89zJ16GYdL6KirEtaG2NUGpZrn9gFxu6/N+PQ49/bl26kacVcc3ueLEL3Pgk3nAC6CZcIofcpxylo/bdAKJfBLzKB0XXL5zlle+2PnHxs6mhYD7M+1GXTLlHcfgNNiTGa+XuvTYq/j9zyXRLIBvMT7zo36Sgxf/olzPTLDDIwpVBS3nidhZrKqiPBPd26YswWmKOEjGCsX+jmzkkiuUo6NM6DPpN4T2aVXJFvnLLZ5ay8= ceph-004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:40:56.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:40:54.352253+0000 mgr.smithi121.otezso (mgr.14116) 5 : cephadm [INF] Generating ssh key... 2024-04-23T00:40:56.338 INFO:teuthology.orchestra.run.smithi121.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-23T00:40:56.339 INFO:teuthology.orchestra.run.smithi121.stderr:Adding key to root@localhost authorized_keys... 2024-04-23T00:40:56.339 INFO:teuthology.orchestra.run.smithi121.stderr:Adding host smithi121... 2024-04-23T00:40:56.339 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch host add smithi121 2024-04-23T00:40:57.092 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:55.814318+0000 mgr.smithi121.otezso (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:40:57.566 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Added host 'smithi121' 2024-04-23T00:40:58.229 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying mon service with default placement... 2024-04-23T00:40:58.229 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply mon 2024-04-23T00:40:58.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:57.008124+0000 mgr.smithi121.otezso (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi121", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:40:58.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:57.574125+0000 mon.smithi121 (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:40:58.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:40:57.574417+0000 mgr.smithi121.otezso (mgr.14116) 8 : cephadm [INF] Added host smithi121 2024-04-23T00:40:58.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:40:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:57.574657+0000 mon.smithi121 (mon.0) 56 : audit [DBG] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:40:59.174 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-23T00:40:59.804 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying mgr service with default placement... 2024-04-23T00:40:59.805 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply mgr 2024-04-23T00:41:00.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:59.180343+0000 mgr.smithi121.otezso (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:00.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:40:59.181018+0000 mgr.smithi121.otezso (mgr.14116) 10 : cephadm [INF] Saving service mon spec with placement count:5 2024-04-23T00:41:00.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:40:59.183102+0000 mon.smithi121 (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:00.646 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-23T00:41:01.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:00.649561+0000 mgr.smithi121.otezso (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:01.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:00.650242+0000 mgr.smithi121.otezso (mgr.14116) 12 : cephadm [INF] Saving service mgr spec with placement count:2 2024-04-23T00:41:01.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:00.652453+0000 mon.smithi121 (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:01.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:01.546909+0000 mon.smithi121 (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:01.958 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying crash service with default placement... 2024-04-23T00:41:01.959 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply crash 2024-04-23T00:41:02.834 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-23T00:41:03.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:01.757670+0000 mon.smithi121 (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:03.636 INFO:teuthology.orchestra.run.smithi121.stderr:Enabling mgr prometheus module... 2024-04-23T00:41:03.636 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr module enable prometheus 2024-04-23T00:41:03.899 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:02.840680+0000 mgr.smithi121.otezso (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:03.899 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:02.841309+0000 mgr.smithi121.otezso (mgr.14116) 14 : cephadm [INF] Saving service crash spec with placement * 2024-04-23T00:41:03.899 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:02.843380+0000 mon.smithi121 (mon.0) 61 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:05.310 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:03.957292+0000 mon.smithi121 (mon.0) 62 : audit [INF] from='mgr.14116 172.21.15.121:0/2800973197' entity='mgr.smithi121.otezso' 2024-04-23T00:41:05.310 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:04.381740+0000 mon.smithi121 (mon.0) 63 : audit [INF] from='client.? 172.21.15.121:0/2438914808' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-23T00:41:05.630 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying prometheus service with default placement... 2024-04-23T00:41:05.630 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply prometheus 2024-04-23T00:41:06.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:04.958871+0000 mon.smithi121 (mon.0) 64 : audit [INF] from='client.? 172.21.15.121:0/2438914808' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-23T00:41:06.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:04.958918+0000 mon.smithi121 (mon.0) 65 : cluster [DBG] mgrmap e9: smithi121.otezso(active, since 14s) 2024-04-23T00:41:10.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:09.809956+0000 mon.smithi121 (mon.0) 66 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T00:41:10.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:09.810341+0000 mon.smithi121 (mon.0) 67 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:41:10.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:09.812650+0000 mon.smithi121 (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-04-23T00:41:10.863 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:09.863173+0000 mon.smithi121 (mon.0) 69 : cluster [DBG] mgrmap e10: smithi121.otezso(active, starting, since 0.0529137s) 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.865783+0000 mon.smithi121 (mon.0) 70 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.866355+0000 mon.smithi121 (mon.0) 71 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.866995+0000 mon.smithi121 (mon.0) 72 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.867222+0000 mon.smithi121 (mon.0) 73 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.867431+0000 mon.smithi121 (mon.0) 74 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:09.880670+0000 mon.smithi121 (mon.0) 75 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:41:10.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.979642+0000 mon.smithi121 (mon.0) 76 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:41:10.916 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.981064+0000 mon.smithi121 (mon.0) 77 : audit [DBG] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:41:10.916 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:09.998180+0000 mon.smithi121 (mon.0) 78 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:41:10.916 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:10.006014+0000 mon.smithi121 (mon.0) 79 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:41:11.691 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying grafana service with default placement... 2024-04-23T00:41:11.692 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply grafana 2024-04-23T00:41:11.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:10.867589+0000 mon.smithi121 (mon.0) 80 : cluster [DBG] mgrmap e11: smithi121.otezso(active, since 1.05733s) 2024-04-23T00:41:11.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:10.868286+0000 mgr.smithi121.otezso (mgr.14142) 2 : cephadm [INF] Saving service prometheus spec with placement count:1 2024-04-23T00:41:11.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:10.871626+0000 mon.smithi121 (mon.0) 81 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' 2024-04-23T00:41:11.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:11.349648+0000 mon.smithi121 (mon.0) 82 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' 2024-04-23T00:41:12.988 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-23T00:41:13.560 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:12.353542+0000 mon.smithi121 (mon.0) 83 : cluster [DBG] mgrmap e12: smithi121.otezso(active, since 2s) 2024-04-23T00:41:13.560 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:12.994540+0000 mgr.smithi121.otezso (mgr.14142) 3 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:13.560 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:12.995125+0000 mgr.smithi121.otezso (mgr.14142) 4 : cephadm [INF] Saving service grafana spec with placement count:1 2024-04-23T00:41:13.560 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:12.997403+0000 mon.smithi121 (mon.0) 84 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' 2024-04-23T00:41:13.877 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying node-exporter service with default placement... 2024-04-23T00:41:13.877 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply node-exporter 2024-04-23T00:41:14.596 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-23T00:41:15.254 INFO:teuthology.orchestra.run.smithi121.stderr:Deploying alertmanager service with default placement... 2024-04-23T00:41:15.254 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply alertmanager 2024-04-23T00:41:15.867 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:14.602513+0000 mgr.smithi121.otezso (mgr.14142) 5 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:15.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:14.603187+0000 mgr.smithi121.otezso (mgr.14142) 6 : cephadm [INF] Saving service node-exporter spec with placement * 2024-04-23T00:41:15.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:14.605457+0000 mon.smithi121 (mon.0) 85 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' 2024-04-23T00:41:15.978 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-23T00:41:16.637 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mgr mgr/cephadm/container_init True --force 2024-04-23T00:41:17.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:15.984065+0000 mgr.smithi121.otezso (mgr.14142) 7 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:17.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:15.984605+0000 mgr.smithi121.otezso (mgr.14142) 8 : cephadm [INF] Saving service alertmanager spec with placement count:1 2024-04-23T00:41:17.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:15.986887+0000 mon.smithi121 (mon.0) 86 : audit [INF] from='mgr.14142 172.21.15.121:0/555525949' entity='mgr.smithi121.otezso' 2024-04-23T00:41:17.959 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mgr mgr/dashboard/ssl_server_port 8443 2024-04-23T00:41:18.617 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:17.391881+0000 mon.smithi121 (mon.0) 87 : audit [INF] from='client.? 172.21.15.121:0/2105882991' entity='client.admin' 2024-04-23T00:41:19.851 INFO:teuthology.orchestra.run.smithi121.stderr:Enabling the dashboard module... 2024-04-23T00:41:19.851 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr module enable dashboard 2024-04-23T00:41:19.853 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:18.722239+0000 mon.smithi121 (mon.0) 88 : audit [INF] from='client.? 172.21.15.121:0/1572121715' entity='client.admin' 2024-04-23T00:41:21.037 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:20.613562+0000 mon.smithi121 (mon.0) 89 : audit [INF] from='client.? 172.21.15.121:0/1305830106' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-23T00:41:21.316 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr stat 2024-04-23T00:41:21.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:20.729390+0000 mon.smithi121 (mon.0) 90 : audit [INF] from='client.? 172.21.15.121:0/1305830106' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-23T00:41:21.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:20.729440+0000 mon.smithi121 (mon.0) 91 : cluster [DBG] mgrmap e13: smithi121.otezso(active, since 10s) 2024-04-23T00:41:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:41:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "epoch": 13, 2024-04-23T00:41:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "available": true, 2024-04-23T00:41:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "active_name": "smithi121.otezso", 2024-04-23T00:41:22.114 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-23T00:41:22.115 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:41:22.765 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for the mgr to restart... 2024-04-23T00:41:22.765 INFO:teuthology.orchestra.run.smithi121.stderr:Waiting for mgr epoch 13... 2024-04-23T00:41:22.765 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 tell mgr mgr_status 2024-04-23T00:41:22.767 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:22.122540+0000 mon.smithi121 (mon.0) 92 : audit [DBG] from='client.? 172.21.15.121:0/445514223' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-23T00:41:26.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:25.646081+0000 mon.smithi121 (mon.0) 93 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T00:41:26.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:25.646475+0000 mon.smithi121 (mon.0) 94 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:41:26.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:25.648138+0000 mon.smithi121 (mon.0) 95 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-04-23T00:41:26.706 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: { 2024-04-23T00:41:26.706 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "mgrmap_epoch": 15, 2024-04-23T00:41:26.706 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: "initialized": true 2024-04-23T00:41:26.706 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: } 2024-04-23T00:41:26.789 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:25.698917+0000 mon.smithi121 (mon.0) 96 : cluster [DBG] mgrmap e14: smithi121.otezso(active, starting, since 0.052532s) 2024-04-23T00:41:26.789 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.700856+0000 mon.smithi121 (mon.0) 97 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:41:26.789 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.701444+0000 mon.smithi121 (mon.0) 98 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.702083+0000 mon.smithi121 (mon.0) 99 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.702217+0000 mon.smithi121 (mon.0) 100 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.702349+0000 mon.smithi121 (mon.0) 101 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:25.715643+0000 mon.smithi121 (mon.0) 102 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.817931+0000 mon.smithi121 (mon.0) 103 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.820068+0000 mon.smithi121 (mon.0) 104 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.836944+0000 mon.smithi121 (mon.0) 105 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:41:26.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:25.844504+0000 mon.smithi121 (mon.0) 106 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:41:27.558 INFO:teuthology.orchestra.run.smithi121.stderr:mgr epoch 13 is available 2024-04-23T00:41:27.559 INFO:teuthology.orchestra.run.smithi121.stderr:Generating a dashboard self-signed certificate... 2024-04-23T00:41:27.559 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 dashboard create-self-signed-cert 2024-04-23T00:41:27.819 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:26.708496+0000 mgr.smithi121.otezso (mgr.14162) 1 : audit [DBG] from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-23T00:41:27.819 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:26.709089+0000 mon.smithi121 (mon.0) 107 : cluster [DBG] mgrmap e15: smithi121.otezso(active, since 1.0627s) 2024-04-23T00:41:27.819 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:26.714500+0000 mgr.smithi121.otezso (mgr.14162) 2 : audit [DBG] from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-23T00:41:27.819 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:27.197668+0000 mon.smithi121 (mon.0) 108 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:28.559 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-23T00:41:29.277 INFO:teuthology.orchestra.run.smithi121.stderr:Creating initial admin user... 2024-04-23T00:41:29.277 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpgc9paha2:/tmp/dashboard.pw:z quay.io/ceph/ceph:v16.2.0 dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2024-04-23T00:41:29.280 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:28.199064+0000 mon.smithi121 (mon.0) 109 : cluster [DBG] mgrmap e16: smithi121.otezso(active, since 2s) 2024-04-23T00:41:29.280 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:28.502491+0000 mgr.smithi121.otezso (mgr.14162) 3 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:29.280 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:28.565709+0000 mon.smithi121 (mon.0) 110 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:29.280 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:28.567387+0000 mon.smithi121 (mon.0) 111 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:30.473 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$iVfg2VLMsQiRjIkYoFCEmOwwr6qvp8rhfzwpozuzynitmGqhPz7dO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713832890, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-23T00:41:31.180 INFO:teuthology.orchestra.run.smithi121.stderr:Fetching dashboard port number... 2024-04-23T00:41:31.180 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi121 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmprbd55yz3:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpv_luuy1c:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config get mgr mgr/dashboard/ssl_server_port 2024-04-23T00:41:31.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:30.263445+0000 mgr.smithi121.otezso (mgr.14162) 4 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:31.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:30.482066+0000 mon.smithi121 (mon.0) 112 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:31.891 INFO:teuthology.orchestra.run.smithi121.stderr:/usr/bin/ceph: 8443 2024-04-23T00:41:32.618 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:31.900452+0000 mon.smithi121 (mon.0) 113 : audit [DBG] from='client.? 172.21.15.121:0/1386443849' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-23T00:41:32.629 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-enabled firewalld.service 2024-04-23T00:41:32.638 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: disabled 2024-04-23T00:41:32.638 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: systemctl is-active firewalld.service 2024-04-23T00:41:32.645 INFO:teuthology.orchestra.run.smithi121.stderr:systemctl: inactive 2024-04-23T00:41:32.646 INFO:teuthology.orchestra.run.smithi121.stderr:firewalld.service is not enabled 2024-04-23T00:41:32.646 INFO:teuthology.orchestra.run.smithi121.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr:Ceph Dashboard is now available at: 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr: URL: https://smithi121.front.sepia.ceph.com:8443/ 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr: User: admin 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr: Password: 1wxg8ulhwu 2024-04-23T00:41:32.649 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr:You can access the Ceph CLI with: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: ceph telemetry on 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.650 INFO:teuthology.orchestra.run.smithi121.stderr:For more information see: 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr: https://docs.ceph.com/docs/pacific/mgr/telemetry/ 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr: 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr:Bootstrap complete. 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr:Releasing lock 140438534033536 on /run/cephadm/004558ac-010a-11ef-bc93-c7b262605968.lock 2024-04-23T00:41:32.651 INFO:teuthology.orchestra.run.smithi121.stderr:Lock 140438534033536 released on /run/cephadm/004558ac-010a-11ef-bc93-c7b262605968.lock 2024-04-23T00:41:32.682 INFO:tasks.cephadm:Fetching config... 2024-04-23T00:41:32.682 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:41:32.682 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-23T00:41:32.701 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-23T00:41:32.701 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:41:32.701 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-23T00:41:32.759 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-23T00:41:32.759 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:41:32.759 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/keyring of=/dev/stdout 2024-04-23T00:41:32.826 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-23T00:41:32.827 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:41:32.827 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-23T00:41:32.885 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-23T00:41:32.885 DEBUG:teuthology.orchestra.run.smithi121:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCa5zrw39+PxwD4/TOriluqagfVHileNnsYK+1H/Q3yzI+tEdcDo61nEAmsu/GN2UewTNDewcF0l0CMcoTw3OPq3wFqQLYhFC/BneIVJqvDsbeEbq2doL7ZIELZYGHZoMaVpbkCTUBsXhjBne75Kgdif0FFuEofSbsVbgHYChDF2mjOJvmizEpVjgGUGGs/hsmL2pg1+8Z8SwypVpg5fHjW8mYnITuBC5KIfK/wHJvSynby07nGWjeZUd/S/ve62tNlMpo9Nap5iywwbm7CAi89zJ16GYdL6KirEtaG2NUGpZrn9gFxu6/N+PQ49/bl26kacVcc3ueLEL3Pgk3nAC6CZcIofcpxylo/bdAKJfBLzKB0XXL5zlle+2PnHxs6mhYD7M+1GXTLlHcfgNNiTGa+XuvTYq/j9zyXRLIBvMT7zo36Sgxf/olzPTLDDIwpVBS3nidhZrKqiPBPd26YswWmKOEjGCsX+jmzkkiuUo6NM6DPpN4T2aVXJFvnLLZ5ay8= ceph-004558ac-010a-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-23T00:41:32.962 INFO:teuthology.orchestra.run.smithi121.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCa5zrw39+PxwD4/TOriluqagfVHileNnsYK+1H/Q3yzI+tEdcDo61nEAmsu/GN2UewTNDewcF0l0CMcoTw3OPq3wFqQLYhFC/BneIVJqvDsbeEbq2doL7ZIELZYGHZoMaVpbkCTUBsXhjBne75Kgdif0FFuEofSbsVbgHYChDF2mjOJvmizEpVjgGUGGs/hsmL2pg1+8Z8SwypVpg5fHjW8mYnITuBC5KIfK/wHJvSynby07nGWjeZUd/S/ve62tNlMpo9Nap5iywwbm7CAi89zJ16GYdL6KirEtaG2NUGpZrn9gFxu6/N+PQ49/bl26kacVcc3ueLEL3Pgk3nAC6CZcIofcpxylo/bdAKJfBLzKB0XXL5zlle+2PnHxs6mhYD7M+1GXTLlHcfgNNiTGa+XuvTYq/j9zyXRLIBvMT7zo36Sgxf/olzPTLDDIwpVBS3nidhZrKqiPBPd26YswWmKOEjGCsX+jmzkkiuUo6NM6DPpN4T2aVXJFvnLLZ5ay8= ceph-004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:41:32.974 DEBUG:teuthology.orchestra.run.smithi153:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCa5zrw39+PxwD4/TOriluqagfVHileNnsYK+1H/Q3yzI+tEdcDo61nEAmsu/GN2UewTNDewcF0l0CMcoTw3OPq3wFqQLYhFC/BneIVJqvDsbeEbq2doL7ZIELZYGHZoMaVpbkCTUBsXhjBne75Kgdif0FFuEofSbsVbgHYChDF2mjOJvmizEpVjgGUGGs/hsmL2pg1+8Z8SwypVpg5fHjW8mYnITuBC5KIfK/wHJvSynby07nGWjeZUd/S/ve62tNlMpo9Nap5iywwbm7CAi89zJ16GYdL6KirEtaG2NUGpZrn9gFxu6/N+PQ49/bl26kacVcc3ueLEL3Pgk3nAC6CZcIofcpxylo/bdAKJfBLzKB0XXL5zlle+2PnHxs6mhYD7M+1GXTLlHcfgNNiTGa+XuvTYq/j9zyXRLIBvMT7zo36Sgxf/olzPTLDDIwpVBS3nidhZrKqiPBPd26YswWmKOEjGCsX+jmzkkiuUo6NM6DPpN4T2aVXJFvnLLZ5ay8= ceph-004558ac-010a-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-23T00:41:33.019 INFO:teuthology.orchestra.run.smithi153.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCa5zrw39+PxwD4/TOriluqagfVHileNnsYK+1H/Q3yzI+tEdcDo61nEAmsu/GN2UewTNDewcF0l0CMcoTw3OPq3wFqQLYhFC/BneIVJqvDsbeEbq2doL7ZIELZYGHZoMaVpbkCTUBsXhjBne75Kgdif0FFuEofSbsVbgHYChDF2mjOJvmizEpVjgGUGGs/hsmL2pg1+8Z8SwypVpg5fHjW8mYnITuBC5KIfK/wHJvSynby07nGWjeZUd/S/ve62tNlMpo9Nap5iywwbm7CAi89zJ16GYdL6KirEtaG2NUGpZrn9gFxu6/N+PQ49/bl26kacVcc3ueLEL3Pgk3nAC6CZcIofcpxylo/bdAKJfBLzKB0XXL5zlle+2PnHxs6mhYD7M+1GXTLlHcfgNNiTGa+XuvTYq/j9zyXRLIBvMT7zo36Sgxf/olzPTLDDIwpVBS3nidhZrKqiPBPd26YswWmKOEjGCsX+jmzkkiuUo6NM6DPpN4T2aVXJFvnLLZ5ay8= ceph-004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:41:33.031 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-23T00:41:33.807 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:32.486580+0000 mon.smithi121 (mon.0) 114 : cluster [DBG] mgrmap e17: smithi121.otezso(active, since 6s) 2024-04-23T00:41:36.701 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi153 2024-04-23T00:41:36.701 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:41:36.702 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.conf 2024-04-23T00:41:36.723 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:41:36.723 DEBUG:teuthology.orchestra.run.smithi153:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-23T00:41:36.783 INFO:tasks.cephadm:Adding host smithi153 to orchestrator... 2024-04-23T00:41:36.784 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch host add smithi153 2024-04-23T00:41:37.072 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:35.639255+0000 mon.smithi121 (mon.0) 115 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:37.072 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:35.788130+0000 mon.smithi121 (mon.0) 116 : audit [INF] from='client.? 172.21.15.121:0/951786723' entity='client.admin' 2024-04-23T00:41:37.072 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:35.963777+0000 mon.smithi121 (mon.0) 117 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:37.675 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:35.970163+0000 mgr.smithi121.otezso (mgr.14162) 5 : cephadm [INF] Deploying daemon alertmanager.smithi121 on smithi121 2024-04-23T00:41:41.073 INFO:teuthology.orchestra.run.smithi121.stdout:Added host 'smithi153' 2024-04-23T00:41:41.094 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:40.485775+0000 mgr.smithi121.otezso (mgr.14162) 6 : audit [DBG] from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi153", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:41.708 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-23T00:41:42.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:41.081300+0000 mon.smithi121 (mon.0) 118 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:42.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:41.081599+0000 mgr.smithi121.otezso (mgr.14162) 7 : cephadm [INF] Added host smithi153 2024-04-23T00:41:43.557 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:41:43.557 INFO:teuthology.orchestra.run.smithi121.stdout:[{"addr": "smithi121", "hostname": "smithi121", "labels": [], "status": ""}, {"addr": "smithi153", "hostname": "smithi153", "labels": [], "status": ""}] 2024-04-23T00:41:44.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:43.565774+0000 mgr.smithi121.otezso (mgr.14162) 8 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:41:44.135 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-23T00:41:44.135 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-23T00:41:46.705 INFO:teuthology.orchestra.run.smithi121.stderr:adjusted tunables profile to default 2024-04-23T00:41:47.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:45.703338+0000 mgr.smithi121.otezso (mgr.14162) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:47.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:45.974130+0000 mon.smithi121 (mon.0) 119 : audit [INF] from='client.? 172.21.15.121:0/1409183063' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-23T00:41:47.529 INFO:tasks.cephadm:Adding mon.smithi121 on smithi121 2024-04-23T00:41:47.530 INFO:tasks.cephadm:Adding mon.smithi153 on smithi153 2024-04-23T00:41:47.530 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi121:172.21.15.121=smithi121;smithi153:172.21.15.153=smithi153' 2024-04-23T00:41:47.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:46.708978+0000 mon.smithi121 (mon.0) 120 : audit [INF] from='client.? 172.21.15.121:0/1409183063' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-23T00:41:47.868 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:46.709041+0000 mon.smithi121 (mon.0) 121 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-04-23T00:41:49.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:47.703565+0000 mgr.smithi121.otezso (mgr.14162) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:49.477 INFO:teuthology.orchestra.run.smithi153.stdout:Scheduled mon update... 2024-04-23T00:41:50.065 DEBUG:teuthology.orchestra.run.smithi153:mon.smithi153> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi153.service 2024-04-23T00:41:50.067 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:41:50.068 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:41:50.097 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:-- Logs begin at Tue 2024-04-23 00:33:27 UTC. -- 2024-04-23T00:41:50.724 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:49.480675+0000 mgr.smithi121.otezso (mgr.14162) 11 : audit [DBG] from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi121:172.21.15.121=smithi121;smithi153:172.21.15.153=smithi153", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:41:50.724 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:49.481953+0000 mgr.smithi121.otezso (mgr.14162) 12 : cephadm [INF] Saving service mon spec with placement smithi121:172.21.15.121=smithi121;smithi153:172.21.15.153=smithi153;count:2 2024-04-23T00:41:50.724 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:49.484456+0000 mon.smithi121 (mon.0) 122 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:50.724 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:49.703856+0000 mgr.smithi121.otezso (mgr.14162) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:52.056 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:41:52.057 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:41:52.058 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:41:53.073 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:51.704156+0000 mgr.smithi121.otezso (mgr.14162) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:53.073 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:52.064658+0000 mon.smithi121 (mon.0) 123 : audit [DBG] from='client.? 172.21.15.153:0/2152962936' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:41:53.788 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:41:53.788 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:53.211019+0000 mon.smithi121 (mon.0) 124 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:53.212042+0000 mon.smithi121 (mon.0) 125 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:53.215940+0000 mon.smithi121 (mon.0) 126 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:53.216524+0000 mon.smithi121 (mon.0) 127 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:53.217077+0000 mgr.smithi121.otezso (mgr.14162) 15 : cephadm [INF] Deploying daemon crash.smithi121 on smithi121 2024-04-23T00:41:54.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:53.704448+0000 mgr.smithi121.otezso (mgr.14162) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:55.743 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:41:55.743 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:41:55.744 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.319467+0000 mon.smithi121 (mon.0) 128 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.453794+0000 mon.smithi121 (mon.0) 129 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.455510+0000 mon.smithi121 (mon.0) 130 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.455903+0000 mon.smithi121 (mon.0) 131 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.456312+0000 mgr.smithi121.otezso (mgr.14162) 17 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.458539+0000 mon.smithi121 (mon.0) 132 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:41:55.461752+0000 mgr.smithi121.otezso (mgr.14162) 18 : cephadm [INF] Deploying daemon grafana.smithi121 on smithi121 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:55.704691+0000 mgr.smithi121.otezso (mgr.14162) 19 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:56.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:55.750785+0000 mon.smithi121 (mon.0) 133 : audit [DBG] from='client.? 172.21.15.153:0/2038214171' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:41:57.356 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:41:57.356 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:41:59.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:57.705044+0000 mgr.smithi121.otezso (mgr.14162) 20 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:41:59.325 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:41:59.325 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:41:59.327 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:00.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:41:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:41:59.332971+0000 mon.smithi121 (mon.0) 134 : audit [DBG] from='client.? 172.21.15.153:0/2329871551' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:00.980 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:00.980 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:01.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:41:59.705442+0000 mgr.smithi121.otezso (mgr.14162) 21 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:03.009 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:03.009 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:03.010 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:03.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:01.705713+0000 mgr.smithi121.otezso (mgr.14162) 22 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:04.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:03.016466+0000 mon.smithi121 (mon.0) 135 : audit [DBG] from='client.? 172.21.15.153:0/662290958' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:04.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:03.705959+0000 mgr.smithi121.otezso (mgr.14162) 23 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:04.719 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:04.719 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:06.733 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:06.734 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:06.735 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:07.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:05.706149+0000 mgr.smithi121.otezso (mgr.14162) 24 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:07.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:06.741445+0000 mon.smithi121 (mon.0) 136 : audit [DBG] from='client.? 172.21.15.153:0/4018528051' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:08.345 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:08.345 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:09.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:07.706329+0000 mgr.smithi121.otezso (mgr.14162) 25 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:10.288 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:10.288 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:10.289 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:10.929 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:09.706622+0000 mgr.smithi121.otezso (mgr.14162) 26 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:10.929 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:10.295083+0000 mon.smithi121 (mon.0) 137 : audit [DBG] from='client.? 172.21.15.153:0/2523996401' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:11.984 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:11.984 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:12.857 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:11.707010+0000 mgr.smithi121.otezso (mgr.14162) 27 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:14.102 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:14.102 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:14.103 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:14.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:12.989722+0000 mon.smithi121 (mon.0) 138 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:14.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:42:12.990921+0000 mgr.smithi121.otezso (mgr.14162) 28 : cephadm [INF] Deploying daemon node-exporter.smithi121 on smithi121 2024-04-23T00:42:14.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:13.707404+0000 mgr.smithi121.otezso (mgr.14162) 29 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:15.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:14.109565+0000 mon.smithi121 (mon.0) 139 : audit [DBG] from='client.? 172.21.15.153:0/1010814179' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:15.681 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:15.681 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:16.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:15.707952+0000 mgr.smithi121.otezso (mgr.14162) 30 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:17.673 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:17.673 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:17.674 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:18.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:17.680477+0000 mon.smithi121 (mon.0) 140 : audit [DBG] from='client.? 172.21.15.153:0/3905404936' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:19.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:17.708320+0000 mgr.smithi121.otezso (mgr.14162) 31 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:19.368 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:19.369 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:21.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:19.708640+0000 mgr.smithi121.otezso (mgr.14162) 32 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:21.362 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:21.363 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:21.364 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:22.090 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:21.370234+0000 mon.smithi121 (mon.0) 141 : audit [DBG] from='client.? 172.21.15.153:0/1299176773' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:22.869 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:21.708885+0000 mgr.smithi121.otezso (mgr.14162) 33 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:23.031 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:23.031 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:24.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:22.941292+0000 mon.smithi121 (mon.0) 142 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:24.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:42:22.945803+0000 mgr.smithi121.otezso (mgr.14162) 34 : cephadm [INF] Deploying daemon prometheus.smithi121 on smithi121 2024-04-23T00:42:24.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:23.709157+0000 mgr.smithi121.otezso (mgr.14162) 35 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:25.141 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:25.142 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:25.143 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:25.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:25.148874+0000 mon.smithi121 (mon.0) 143 : audit [DBG] from='client.? 172.21.15.153:0/1101124641' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:26.619 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:25.709423+0000 mgr.smithi121.otezso (mgr.14162) 36 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:26.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:25.838910+0000 mon.smithi121 (mon.0) 144 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:42:26.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:25.847208+0000 mon.smithi121 (mon.0) 145 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:42:26.804 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:26.804 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:28.696 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:28.697 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:28.698 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:29.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:27.709764+0000 mgr.smithi121.otezso (mgr.14162) 37 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:29.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:28.703744+0000 mon.smithi121 (mon.0) 146 : audit [DBG] from='client.? 172.21.15.153:0/2507030884' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:30.342 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:30.343 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:31.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:29.710176+0000 mgr.smithi121.otezso (mgr.14162) 38 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:32.258 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:32.258 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:32.259 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:33.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:31.710488+0000 mgr.smithi121.otezso (mgr.14162) 39 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:33.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:32.264952+0000 mon.smithi121 (mon.0) 147 : audit [DBG] from='client.? 172.21.15.153:0/1281168462' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:33.865 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:33.865 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:34.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:33.710834+0000 mgr.smithi121.otezso (mgr.14162) 40 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:35.915 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:35.915 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:35.916 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:36.869 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:35.711064+0000 mgr.smithi121.otezso (mgr.14162) 41 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:36.869 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:35.921511+0000 mon.smithi121 (mon.0) 148 : audit [DBG] from='client.? 172.21.15.153:0/589257978' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:37.616 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:37.616 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:39.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:37.711456+0000 mgr.smithi121.otezso (mgr.14162) 42 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:39.579 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:39.579 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:39.580 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:39.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:39.585880+0000 mon.smithi121 (mon.0) 149 : audit [DBG] from='client.? 172.21.15.153:0/3067429480' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:41.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:39.711714+0000 mgr.smithi121.otezso (mgr.14162) 43 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:41.161 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:41.161 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:43.025 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:41.712025+0000 mgr.smithi121.otezso (mgr.14162) 44 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:43.025 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:42.275722+0000 mon.smithi121 (mon.0) 150 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:43.025 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:42.276345+0000 mon.smithi121 (mon.0) 151 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:42:43.025 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:42.493501+0000 mon.smithi121 (mon.0) 152 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:43.133 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:43.133 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:43.136 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:44.370 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:42.950718+0000 mon.smithi121 (mon.0) 153 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:44.370 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:43.140568+0000 mon.smithi121 (mon.0) 154 : audit [DBG] from='client.? 172.21.15.153:0/4009764698' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:44.370 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:43.712370+0000 mgr.smithi121.otezso (mgr.14162) 45 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:45.573 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:45.573 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:47.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:45.712638+0000 mgr.smithi121.otezso (mgr.14162) 46 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:47.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:46.333234+0000 mon.smithi121 (mon.0) 155 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:48.190 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:48.191 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:48.193 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.193331+0000 mon.smithi121 (mon.0) 156 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.451631+0000 mon.smithi121 (mon.0) 157 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.454301+0000 mon.smithi121 (mon.0) 158 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.455411+0000 mon.smithi121 (mon.0) 159 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.457320+0000 mon.smithi121 (mon.0) 160 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:47.458021+0000 mon.smithi121 (mon.0) 161 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:42:47.458757+0000 mgr.smithi121.otezso (mgr.14162) 47 : cephadm [INF] Deploying daemon crash.smithi153 on smithi153 2024-04-23T00:42:48.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:47.712969+0000 mgr.smithi121.otezso (mgr.14162) 48 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:49.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:48.197737+0000 mon.smithi121 (mon.0) 162 : audit [DBG] from='client.? 172.21.15.153:0/3716382676' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:50.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:49.713357+0000 mgr.smithi121.otezso (mgr.14162) 49 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:50.767 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:50.767 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:51.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:50.334711+0000 mon.smithi121 (mon.0) 163 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:51.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:50.335981+0000 mon.smithi121 (mon.0) 164 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:42:51.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:50.338660+0000 mon.smithi121 (mon.0) 165 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-23T00:42:51.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:50.339395+0000 mon.smithi121 (mon.0) 166 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:42:51.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:50.340228+0000 mon.smithi121 (mon.0) 167 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:42:51.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:42:50.340862+0000 mgr.smithi121.otezso (mgr.14162) 50 : cephadm [INF] Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T00:42:52.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:51.713583+0000 mgr.smithi121.otezso (mgr.14162) 51 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:53.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:52.408126+0000 mon.smithi121 (mon.0) 168 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:42:53.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:52.409442+0000 mon.smithi121 (mon.0) 169 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T00:42:53.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:52.410275+0000 mon.smithi121 (mon.0) 170 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:42:53.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:42:52.411074+0000 mgr.smithi121.otezso (mgr.14162) 52 : cephadm [INF] Deploying daemon mon.smithi153 on smithi153 2024-04-23T00:42:54.057 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:42:54.057 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":1,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:40:20.161921Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-23T00:42:54.059 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 1 2024-04-23T00:42:54.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:53.713855+0000 mgr.smithi121.otezso (mgr.14162) 53 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:54.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:54.064085+0000 mon.smithi121 (mon.0) 171 : audit [DBG] from='client.? 172.21.15.153:0/4160168703' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:42:55.777 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-23T00:42:55.777 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-23T00:42:57.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:42:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:55.714195+0000 mgr.smithi121.otezso (mgr.14162) 54 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:57.217 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.222+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).mds e1 new map 2024-04-23T00:42:57.217 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.222+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).mds e1 print_map 2024-04-23T00:42:57.217 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: e1 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: legacy client fscid: -1 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: No filesystems configured 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: 2024-04-23T00:42:57.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.222+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 0 mon.smithi153@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:42:55.714195+0000 mgr.smithi121.otezso (mgr.14162) 54 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.223+0000 7f5189488700 1 mon.smithi153@-1(synchronizing).paxosservice(auth 1..7) refresh upgraded, format 0 -> 3 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expand map: {default=false} 2024-04-23T00:42:57.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta from 'false' to 'false' 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expanded map: {default=false} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expand map: {default=info} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta from 'info' to 'info' 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expanded map: {default=info} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expand map: {default=daemon} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta from 'daemon' to 'daemon' 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expanded map: {default=daemon} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expand map: {default=debug} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta from 'debug' to 'debug' 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 20 expand_channel_meta expanded map: {default=debug} 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T00:42:57.225+0000 7f5189488700 10 mon.smithi153@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 bash[38970]: 92021031a7510ad0d59039ca27b8ee425e68735f2d1537a4772c74d5318bc2ab 2024-04-23T00:42:57.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:42:57 smithi153 systemd[1]: Started Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T00:43:02.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:57.233918+0000 mon.smithi121 (mon.0) 173 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:57.234081+0000 mon.smithi121 (mon.0) 174 : cluster [INF] mon.smithi121 calling monitor election 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:57.250140+0000 mon.smithi121 (mon.0) 175 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:57.332265+0000 mon.smithi121 (mon.0) 176 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:57.714551+0000 mgr.smithi121.otezso (mgr.14162) 55 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:58.228385+0000 mon.smithi121 (mon.0) 177 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:42:59.228390+0000 mon.smithi121 (mon.0) 178 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:59.229609+0000 mon.smithi153 (mon.1) 1 : cluster [INF] mon.smithi153 calling monitor election 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:42:59.714885+0000 mgr.smithi121.otezso (mgr.14162) 56 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:00.228675+0000 mon.smithi121 (mon.0) 179 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:01.228755+0000 mon.smithi121 (mon.0) 180 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:01.715204+0000 mgr.smithi121.otezso (mgr.14162) 57 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.228924+0000 mon.smithi121 (mon.0) 181 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.263548+0000 mon.smithi121 (mon.0) 182 : cluster [INF] mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.268348+0000 mon.smithi121 (mon.0) 183 : cluster [DBG] monmap e2: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.268391+0000 mon.smithi121 (mon.0) 184 : cluster [DBG] fsmap 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.268415+0000 mon.smithi121 (mon.0) 185 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.268614+0000 mon.smithi121 (mon.0) 186 : cluster [DBG] mgrmap e17: smithi121.otezso(active, since 96s) 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.269293+0000 mon.smithi121 (mon.0) 187 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.270653+0000 mon.smithi121 (mon.0) 188 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.273269+0000 mon.smithi121 (mon.0) 189 : cluster [INF] overall HEALTH_OK 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.274579+0000 mon.smithi121 (mon.0) 190 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.275004+0000 mon.smithi121 (mon.0) 191 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:43:02.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.278366+0000 mon.smithi121 (mon.0) 192 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:42:57.233918+0000 mon.smithi121 (mon.0) 173 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:42:57.234081+0000 mon.smithi121 (mon.0) 174 : cluster [INF] mon.smithi121 calling monitor election 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:42:57.250140+0000 mon.smithi121 (mon.0) 175 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:42:57.332265+0000 mon.smithi121 (mon.0) 176 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:42:57.714551+0000 mgr.smithi121.otezso (mgr.14162) 55 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:42:58.228385+0000 mon.smithi121 (mon.0) 177 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:42:59.228390+0000 mon.smithi121 (mon.0) 178 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:42:59.229609+0000 mon.smithi153 (mon.1) 1 : cluster [INF] mon.smithi153 calling monitor election 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:42:59.714885+0000 mgr.smithi121.otezso (mgr.14162) 56 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:00.228675+0000 mon.smithi121 (mon.0) 179 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:01.228755+0000 mon.smithi121 (mon.0) 180 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:01.715204+0000 mgr.smithi121.otezso (mgr.14162) 57 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.228924+0000 mon.smithi121 (mon.0) 181 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.263548+0000 mon.smithi121 (mon.0) 182 : cluster [INF] mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.268348+0000 mon.smithi121 (mon.0) 183 : cluster [DBG] monmap e2: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.268391+0000 mon.smithi121 (mon.0) 184 : cluster [DBG] fsmap 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.268415+0000 mon.smithi121 (mon.0) 185 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.268614+0000 mon.smithi121 (mon.0) 186 : cluster [DBG] mgrmap e17: smithi121.otezso(active, since 96s) 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.269293+0000 mon.smithi121 (mon.0) 187 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.270653+0000 mon.smithi121 (mon.0) 188 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.273269+0000 mon.smithi121 (mon.0) 189 : cluster [INF] overall HEALTH_OK 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.274579+0000 mon.smithi121 (mon.0) 190 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.275004+0000 mon.smithi121 (mon.0) 191 : audit [DBG] from='mgr.? 172.21.15.153:0/2732861840' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:43:02.634 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.278366+0000 mon.smithi121 (mon.0) 192 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:03.061 INFO:teuthology.orchestra.run.smithi153.stdout: 2024-04-23T00:43:03.061 INFO:teuthology.orchestra.run.smithi153.stdout:{"epoch":2,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","modified":"2024-04-23T00:42:57.228658Z","created":"2024-04-23T00:40:20.161921Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi121","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:3300","nonce":0},{"type":"v1","addr":"172.21.15.121:6789","nonce":0}]},"addr":"172.21.15.121:6789/0","public_addr":"172.21.15.121:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi153","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:3300","nonce":0},{"type":"v1","addr":"172.21.15.153:6789","nonce":0}]},"addr":"172.21.15.153:6789/0","public_addr":"172.21.15.153:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-23T00:43:03.062 INFO:teuthology.orchestra.run.smithi153.stderr:dumped monmap epoch 2 2024-04-23T00:43:03.435 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:02.279340+0000 mgr.smithi121.otezso (mgr.14162) 58 : cephadm [INF] Deploying daemon node-exporter.smithi153 on smithi153 2024-04-23T00:43:03.435 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:02.331015+0000 mon.smithi121 (mon.0) 193 : cluster [DBG] mgrmap e18: smithi121.otezso(active, since 96s), standbys: smithi153.olhmgd 2024-04-23T00:43:03.435 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:02.331202+0000 mon.smithi121 (mon.0) 194 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:43:03.435 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:03.066180+0000 mon.smithi121 (mon.0) 195 : audit [DBG] from='client.? 172.21.15.153:0/3004783896' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:43:03.435 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:03.229268+0000 mon.smithi121 (mon.0) 196 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:03.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:02.279340+0000 mgr.smithi121.otezso (mgr.14162) 58 : cephadm [INF] Deploying daemon node-exporter.smithi153 on smithi153 2024-04-23T00:43:03.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:02.331015+0000 mon.smithi121 (mon.0) 193 : cluster [DBG] mgrmap e18: smithi121.otezso(active, since 96s), standbys: smithi153.olhmgd 2024-04-23T00:43:03.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:02.331202+0000 mon.smithi121 (mon.0) 194 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:43:03.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:03.066180+0000 mon.smithi121 (mon.0) 195 : audit [DBG] from='client.? 172.21.15.153:0/3004783896' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-23T00:43:03.620 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:03.229268+0000 mon.smithi121 (mon.0) 196 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:43:03.775 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-23T00:43:03.776 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-23T00:43:04.486 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:03.715571+0000 mgr.smithi121.otezso (mgr.14162) 59 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:04.815 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:03.715571+0000 mgr.smithi121.otezso (mgr.14162) 59 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:05.678 INFO:teuthology.orchestra.run.smithi121.stdout:# minimal ceph.conf for 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:43:05.678 INFO:teuthology.orchestra.run.smithi121.stdout:[global] 2024-04-23T00:43:05.678 INFO:teuthology.orchestra.run.smithi121.stdout: fsid = 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T00:43:05.678 INFO:teuthology.orchestra.run.smithi121.stdout: mon_host = [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] 2024-04-23T00:43:06.065 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:05.684463+0000 mon.smithi121 (mon.0) 197 : audit [DBG] from='client.? 172.21.15.121:0/610801105' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:06.077 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:05.684463+0000 mon.smithi121 (mon.0) 197 : audit [DBG] from='client.? 172.21.15.121:0/610801105' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:06.386 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-23T00:43:06.386 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:43:06.386 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.conf 2024-04-23T00:43:06.418 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:43:06.418 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-23T00:43:06.486 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:43:06.486 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.conf 2024-04-23T00:43:06.523 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:43:06.523 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-23T00:43:06.594 INFO:tasks.cephadm:Deploying OSDs... 2024-04-23T00:43:06.594 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:43:06.594 DEBUG:teuthology.orchestra.run.smithi121:> dd if=/scratch_devs of=/dev/stdout 2024-04-23T00:43:06.613 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-23T00:43:06.613 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_1 2024-04-23T00:43:06.671 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-23T00:43:06.671 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:06.671 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 168226 Links: 1 2024-04-23T00:43:06.671 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:06.672 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:06.672 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-23 00:42:39.818655981 +0000 2024-04-23T00:43:06.672 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-23 00:39:06.016200672 +0000 2024-04-23T00:43:06.672 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-23 00:39:06.016200672 +0000 2024-04-23T00:43:06.672 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2024-04-23T00:43:06.672 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-23T00:43:06.738 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-23T00:43:06.738 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-23T00:43:06.738 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000352659 s, 1.5 MB/s 2024-04-23T00:43:06.740 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-23T00:43:06.799 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_2 2024-04-23T00:43:06.815 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:05.715936+0000 mgr.smithi121.otezso (mgr.14162) 60 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 168335 Links: 1 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-23 00:42:39.818655981 +0000 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-23 00:39:06.346196604 +0000 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-23 00:39:06.346196604 +0000 2024-04-23T00:43:06.857 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2024-04-23T00:43:06.858 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-23T00:43:06.923 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-23T00:43:06.923 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-23T00:43:06.923 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000240219 s, 2.1 MB/s 2024-04-23T00:43:06.924 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-23T00:43:06.983 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_3 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 166426 Links: 1 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-23 00:42:39.818655981 +0000 2024-04-23T00:43:07.041 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-23 00:39:06.666192660 +0000 2024-04-23T00:43:07.042 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-23 00:39:06.666192660 +0000 2024-04-23T00:43:07.042 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2024-04-23T00:43:07.042 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-23T00:43:07.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:05.715936+0000 mgr.smithi121.otezso (mgr.14162) 60 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:07.106 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-23T00:43:07.107 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-23T00:43:07.107 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000190492 s, 2.7 MB/s 2024-04-23T00:43:07.107 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-23T00:43:07.167 DEBUG:teuthology.orchestra.run.smithi121:> stat /dev/vg_nvme/lv_4 2024-04-23T00:43:07.225 INFO:teuthology.orchestra.run.smithi121.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-23T00:43:07.225 INFO:teuthology.orchestra.run.smithi121.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:07.225 INFO:teuthology.orchestra.run.smithi121.stdout:Device: 6h/6d Inode: 163809 Links: 1 2024-04-23T00:43:07.225 INFO:teuthology.orchestra.run.smithi121.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:07.225 INFO:teuthology.orchestra.run.smithi121.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:07.226 INFO:teuthology.orchestra.run.smithi121.stdout:Access: 2024-04-23 00:42:39.819655969 +0000 2024-04-23T00:43:07.226 INFO:teuthology.orchestra.run.smithi121.stdout:Modify: 2024-04-23 00:39:07.006188469 +0000 2024-04-23T00:43:07.226 INFO:teuthology.orchestra.run.smithi121.stdout:Change: 2024-04-23 00:39:07.006188469 +0000 2024-04-23T00:43:07.226 INFO:teuthology.orchestra.run.smithi121.stdout: Birth: - 2024-04-23T00:43:07.226 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-23T00:43:07.290 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records in 2024-04-23T00:43:07.290 INFO:teuthology.orchestra.run.smithi121.stderr:1+0 records out 2024-04-23T00:43:07.290 INFO:teuthology.orchestra.run.smithi121.stderr:512 bytes copied, 0.000246152 s, 2.1 MB/s 2024-04-23T00:43:07.291 DEBUG:teuthology.orchestra.run.smithi121:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-23T00:43:07.350 DEBUG:teuthology.orchestra.run.smithi153:> set -ex 2024-04-23T00:43:07.350 DEBUG:teuthology.orchestra.run.smithi153:> dd if=/scratch_devs of=/dev/stdout 2024-04-23T00:43:07.372 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-23T00:43:07.373 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_1 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 164552 Links: 1 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2024-04-23 00:43:02.686747341 +0000 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2024-04-23 00:39:06.011997617 +0000 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2024-04-23 00:39:06.011997617 +0000 2024-04-23T00:43:07.434 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2024-04-23T00:43:07.435 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-23T00:43:07.504 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2024-04-23T00:43:07.504 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2024-04-23T00:43:07.504 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000264412 s, 1.9 MB/s 2024-04-23T00:43:07.505 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-23T00:43:07.568 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_2 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 165651 Links: 1 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2024-04-23 00:43:02.686747341 +0000 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2024-04-23 00:39:06.346991807 +0000 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2024-04-23 00:39:06.346991807 +0000 2024-04-23T00:43:07.631 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2024-04-23T00:43:07.631 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-23T00:43:07.701 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2024-04-23T00:43:07.701 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2024-04-23T00:43:07.701 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000231711 s, 2.2 MB/s 2024-04-23T00:43:07.702 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-23T00:43:07.763 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_3 2024-04-23T00:43:07.822 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-23T00:43:07.822 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 163611 Links: 1 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2024-04-23 00:43:02.686747341 +0000 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2024-04-23 00:39:06.690985841 +0000 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2024-04-23 00:39:06.690985841 +0000 2024-04-23T00:43:07.823 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2024-04-23T00:43:07.823 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-23T00:43:07.890 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2024-04-23T00:43:07.890 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2024-04-23T00:43:07.890 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000206232 s, 2.5 MB/s 2024-04-23T00:43:07.891 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-23T00:43:07.951 DEBUG:teuthology.orchestra.run.smithi153:> stat /dev/vg_nvme/lv_4 2024-04-23T00:43:08.009 INFO:teuthology.orchestra.run.smithi153.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-23T00:43:08.009 INFO:teuthology.orchestra.run.smithi153.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Device: 6h/6d Inode: 165796 Links: 1 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Context: system_u:object_r:device_t:s0 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Access: 2024-04-23 00:43:02.686747341 +0000 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Modify: 2024-04-23 00:39:07.045979684 +0000 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout:Change: 2024-04-23 00:39:07.045979684 +0000 2024-04-23T00:43:08.010 INFO:teuthology.orchestra.run.smithi153.stdout: Birth: - 2024-04-23T00:43:08.010 DEBUG:teuthology.orchestra.run.smithi153:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-23T00:43:08.076 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records in 2024-04-23T00:43:08.076 INFO:teuthology.orchestra.run.smithi153.stderr:1+0 records out 2024-04-23T00:43:08.076 INFO:teuthology.orchestra.run.smithi153.stderr:512 bytes copied, 0.000320724 s, 1.6 MB/s 2024-04-23T00:43:08.077 DEBUG:teuthology.orchestra.run.smithi153:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-23T00:43:08.136 INFO:tasks.cephadm:Deploying osd.0 on smithi121 with /dev/vg_nvme/lv_4... 2024-04-23T00:43:08.136 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-23T00:43:08.538 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:07.716233+0000 mgr.smithi121.otezso (mgr.14162) 61 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:08.565 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:07.716233+0000 mgr.smithi121.otezso (mgr.14162) 61 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112416 s, 933 MB/s 2024-04-23T00:43:10.232 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-23T00:43:10.830 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:09.716461+0000 mgr.smithi121.otezso (mgr.14162) 62 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:10.849 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_4 2024-04-23T00:43:11.065 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:09.716461+0000 mgr.smithi121.otezso (mgr.14162) 62 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:12.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:11.716744+0000 mgr.smithi121.otezso (mgr.14162) 63 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:12.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.086044+0000 mon.smithi121 (mon.0) 198 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:12.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.086784+0000 mon.smithi121 (mon.0) 199 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:43:13.069 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:11.716744+0000 mgr.smithi121.otezso (mgr.14162) 63 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:13.070 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.086044+0000 mon.smithi121 (mon.0) 198 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:13.070 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.086784+0000 mon.smithi121 (mon.0) 199 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:43:14.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.853489+0000 mon.smithi121 (mon.0) 200 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:43:14.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.854871+0000 mon.smithi121 (mon.0) 201 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:43:14.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.855337+0000 mon.smithi121 (mon.0) 202 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:14.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.853489+0000 mon.smithi121 (mon.0) 200 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:43:14.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.854871+0000 mon.smithi121 (mon.0) 201 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:43:14.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.855337+0000 mon.smithi121 (mon.0) 202 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:15.000 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:12.852636+0000 mgr.smithi121.otezso (mgr.14162) 64 : audit [DBG] from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:43:15.000 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:12.854645+0000 mgr.smithi121.otezso (mgr.14162) 65 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:43:15.000 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:13.717138+0000 mgr.smithi121.otezso (mgr.14162) 66 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:15.001 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:13.836721+0000 mon.smithi121 (mon.0) 203 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:15.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:12.852636+0000 mgr.smithi121.otezso (mgr.14162) 64 : audit [DBG] from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:43:15.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:12.854645+0000 mgr.smithi121.otezso (mgr.14162) 65 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:43:15.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:13.717138+0000 mgr.smithi121.otezso (mgr.14162) 66 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:15.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:13.836721+0000 mon.smithi121 (mon.0) 203 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:16.907 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:15.717427+0000 mgr.smithi121.otezso (mgr.14162) 67 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:16.907 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:16.518301+0000 mon.smithi121 (mon.0) 204 : audit [INF] from='client.? 172.21.15.121:0/4199161080' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c518eaa9-6bd6-403b-881b-cca9f381b2f0"}]: dispatch 2024-04-23T00:43:16.908 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:16.521752+0000 mon.smithi121 (mon.0) 205 : audit [INF] from='client.? 172.21.15.121:0/4199161080' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c518eaa9-6bd6-403b-881b-cca9f381b2f0"}]': finished 2024-04-23T00:43:16.908 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:16.521789+0000 mon.smithi121 (mon.0) 206 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-04-23T00:43:16.908 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:16.521885+0000 mon.smithi121 (mon.0) 207 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:17.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:15.717427+0000 mgr.smithi121.otezso (mgr.14162) 67 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:17.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:16.518301+0000 mon.smithi121 (mon.0) 204 : audit [INF] from='client.? 172.21.15.121:0/4199161080' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c518eaa9-6bd6-403b-881b-cca9f381b2f0"}]: dispatch 2024-04-23T00:43:17.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:16.521752+0000 mon.smithi121 (mon.0) 205 : audit [INF] from='client.? 172.21.15.121:0/4199161080' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c518eaa9-6bd6-403b-881b-cca9f381b2f0"}]': finished 2024-04-23T00:43:17.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:16.521789+0000 mon.smithi121 (mon.0) 206 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-04-23T00:43:17.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:16.521885+0000 mon.smithi121 (mon.0) 207 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:18.065 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:16.953600+0000 mon.smithi121 (mon.0) 208 : audit [DBG] from='client.? 172.21.15.121:0/3077509885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:43:18.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:16.953600+0000 mon.smithi121 (mon.0) 208 : audit [DBG] from='client.? 172.21.15.121:0/3077509885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:17.717671+0000 mgr.smithi121.otezso (mgr.14162) 68 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:18.684014+0000 mon.smithi121 (mon.0) 209 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:18.686586+0000 mon.smithi121 (mon.0) 210 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:18.688099+0000 mgr.smithi121.otezso (mgr.14162) 69 : cephadm [INF] Reconfiguring mon.smithi121 (unknown last config time)... 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:18.688308+0000 mon.smithi121 (mon.0) 211 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:18.688785+0000 mon.smithi121 (mon.0) 212 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:18.689221+0000 mon.smithi121 (mon.0) 213 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:19.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:18.689626+0000 mgr.smithi121.otezso (mgr.14162) 70 : cephadm [INF] Reconfiguring daemon mon.smithi121 on smithi121 2024-04-23T00:43:19.120 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:17.717671+0000 mgr.smithi121.otezso (mgr.14162) 68 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:18.684014+0000 mon.smithi121 (mon.0) 209 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:18.686586+0000 mon.smithi121 (mon.0) 210 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:18.688099+0000 mgr.smithi121.otezso (mgr.14162) 69 : cephadm [INF] Reconfiguring mon.smithi121 (unknown last config time)... 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:18.688308+0000 mon.smithi121 (mon.0) 211 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:18.688785+0000 mon.smithi121 (mon.0) 212 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:18.689221+0000 mon.smithi121 (mon.0) 213 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:19.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:18.689626+0000 mgr.smithi121.otezso (mgr.14162) 70 : cephadm [INF] Reconfiguring daemon mon.smithi121 on smithi121 2024-04-23T00:43:20.029 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:19.717963+0000 mgr.smithi121.otezso (mgr.14162) 71 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:20.065 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:19.717963+0000 mgr.smithi121.otezso (mgr.14162) 71 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:21.465594+0000 mon.smithi121 (mon.0) 214 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:21.466051+0000 mgr.smithi121.otezso (mgr.14162) 72 : cephadm [INF] Reconfiguring mgr.smithi121.otezso (unknown last config time)... 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:21.466310+0000 mon.smithi121 (mon.0) 215 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:21.466927+0000 mon.smithi121 (mon.0) 216 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:21.467405+0000 mon.smithi121 (mon.0) 217 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:21.467862+0000 mgr.smithi121.otezso (mgr.14162) 73 : cephadm [INF] Reconfiguring daemon mgr.smithi121.otezso on smithi121 2024-04-23T00:43:22.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:21.718301+0000 mgr.smithi121.otezso (mgr.14162) 74 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:22.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:21.465594+0000 mon.smithi121 (mon.0) 214 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:22.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:21.466051+0000 mgr.smithi121.otezso (mgr.14162) 72 : cephadm [INF] Reconfiguring mgr.smithi121.otezso (unknown last config time)... 2024-04-23T00:43:22.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:21.466310+0000 mon.smithi121 (mon.0) 215 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:43:22.870 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:21.466927+0000 mon.smithi121 (mon.0) 216 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:43:22.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:21.467405+0000 mon.smithi121 (mon.0) 217 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:22.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:21.467862+0000 mgr.smithi121.otezso (mgr.14162) 73 : cephadm [INF] Reconfiguring daemon mgr.smithi121.otezso on smithi121 2024-04-23T00:43:22.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:21.718301+0000 mgr.smithi121.otezso (mgr.14162) 74 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:24.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:23.035881+0000 mon.smithi121 (mon.0) 218 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:24.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:23.036390+0000 mgr.smithi121.otezso (mgr.14162) 75 : cephadm [INF] Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T00:43:24.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:23.038459+0000 mgr.smithi121.otezso (mgr.14162) 76 : cephadm [INF] Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T00:43:24.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:23.718522+0000 mgr.smithi121.otezso (mgr.14162) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:24.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:23.035881+0000 mon.smithi121 (mon.0) 218 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:24.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:23.036390+0000 mgr.smithi121.otezso (mgr.14162) 75 : cephadm [INF] Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T00:43:24.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:23.038459+0000 mgr.smithi121.otezso (mgr.14162) 76 : cephadm [INF] Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T00:43:24.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:23.718522+0000 mgr.smithi121.otezso (mgr.14162) 77 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:25.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:24.426609+0000 mon.smithi121 (mon.0) 219 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T00:43:25.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:24.427206+0000 mon.smithi121 (mon.0) 220 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:25.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:24.427736+0000 mgr.smithi121.otezso (mgr.14162) 78 : cephadm [INF] Deploying daemon osd.0 on smithi121 2024-04-23T00:43:25.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:24.426609+0000 mon.smithi121 (mon.0) 219 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T00:43:25.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:24.427206+0000 mon.smithi121 (mon.0) 220 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:25.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:24.427736+0000 mgr.smithi121.otezso (mgr.14162) 78 : cephadm [INF] Deploying daemon osd.0 on smithi121 2024-04-23T00:43:26.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:25.718903+0000 mgr.smithi121.otezso (mgr.14162) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:26.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:25.840122+0000 mon.smithi121 (mon.0) 221 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:43:26.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:25.847975+0000 mon.smithi121 (mon.0) 222 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:43:26.370 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:25.718903+0000 mgr.smithi121.otezso (mgr.14162) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:26.371 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:25.840122+0000 mon.smithi121 (mon.0) 221 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:43:26.371 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:25.847975+0000 mon.smithi121 (mon.0) 222 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:43:29.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:27.719331+0000 mgr.smithi121.otezso (mgr.14162) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:29.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:27.719331+0000 mgr.smithi121.otezso (mgr.14162) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:29.253411+0000 mon.smithi121 (mon.0) 223 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:29.253856+0000 mgr.smithi121.otezso (mgr.14162) 81 : cephadm [INF] Reconfiguring crash.smithi121 (monmap changed)... 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:29.254064+0000 mon.smithi121 (mon.0) 224 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:29.254648+0000 mon.smithi121 (mon.0) 225 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:29.255053+0000 mgr.smithi121.otezso (mgr.14162) 82 : cephadm [INF] Reconfiguring daemon crash.smithi121 on smithi121 2024-04-23T00:43:30.524 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:29.719741+0000 mgr.smithi121.otezso (mgr.14162) 83 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:29.253411+0000 mon.smithi121 (mon.0) 223 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:29.253856+0000 mgr.smithi121.otezso (mgr.14162) 81 : cephadm [INF] Reconfiguring crash.smithi121 (monmap changed)... 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:29.254064+0000 mon.smithi121 (mon.0) 224 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:29.254648+0000 mon.smithi121 (mon.0) 225 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:29.255053+0000 mgr.smithi121.otezso (mgr.14162) 82 : cephadm [INF] Reconfiguring daemon crash.smithi121 on smithi121 2024-04-23T00:43:30.566 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:29.719741+0000 mgr.smithi121.otezso (mgr.14162) 83 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:32.744 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 0 on host 'smithi121' 2024-04-23T00:43:32.768 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:31.720088+0000 mgr.smithi121.otezso (mgr.14162) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:32.768 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:32.744801+0000 mon.smithi121 (mon.0) 226 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:32.768 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:32.748118+0000 mon.smithi121 (mon.0) 227 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:33.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:31.720088+0000 mgr.smithi121.otezso (mgr.14162) 84 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:33.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:32.744801+0000 mon.smithi121 (mon.0) 226 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:33.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:32.748118+0000 mon.smithi121 (mon.0) 227 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:33.543 DEBUG:teuthology.orchestra.run.smithi121:osd.0> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.0.service 2024-04-23T00:43:33.545 INFO:tasks.cephadm:Deploying osd.1 on smithi121 with /dev/vg_nvme/lv_3... 2024-04-23T00:43:33.545 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-23T00:43:33.574 INFO:journalctl@ceph.osd.0.smithi121.stdout:-- Logs begin at Tue 2024-04-23 00:33:19 UTC. -- 2024-04-23T00:43:34.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:33.720333+0000 mgr.smithi121.otezso (mgr.14162) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:34.371 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:33.720333+0000 mgr.smithi121.otezso (mgr.14162) 85 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:35.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:34.644803+0000 mon.smithi121 (mon.0) 228 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:35.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:34.645307+0000 mgr.smithi121.otezso (mgr.14162) 86 : cephadm [INF] Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T00:43:35.866 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:34.649500+0000 mgr.smithi121.otezso (mgr.14162) 87 : cephadm [INF] Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T00:43:36.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:34.644803+0000 mon.smithi121 (mon.0) 228 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:36.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:34.645307+0000 mgr.smithi121.otezso (mgr.14162) 86 : cephadm [INF] Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T00:43:36.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:34.649500+0000 mgr.smithi121.otezso (mgr.14162) 87 : cephadm [INF] Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T00:43:36.120 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 00:43:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T00:43:35.871+0000 7f7eaf71f080 -1 osd.0 0 log_to_monitors {default=true} 2024-04-23T00:43:36.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:35.720836+0000 mgr.smithi121.otezso (mgr.14162) 88 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:36.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:35.875452+0000 mon.smithi121 (mon.0) 229 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-23T00:43:37.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:35.720836+0000 mgr.smithi121.otezso (mgr.14162) 88 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:37.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:35.875452+0000 mon.smithi121 (mon.0) 229 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111378 s, 941 MB/s 2024-04-23T00:43:37.114 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-23T00:43:37.816 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_3 2024-04-23T00:43:38.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:36.650423+0000 mon.smithi121 (mon.0) 230 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-23T00:43:38.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:36.650468+0000 mon.smithi121 (mon.0) 231 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-04-23T00:43:38.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:36.650630+0000 mon.smithi121 (mon.0) 232 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:38.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:36.650966+0000 mon.smithi121 (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:43:38.121 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T00:43:37.653+0000 7f7ea5c90700 -1 osd.0 0 waiting for initial osdmap 2024-04-23T00:43:38.121 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T00:43:37.658+0000 7f7e9fc07700 -1 osd.0 8 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:43:38.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:36.650423+0000 mon.smithi121 (mon.0) 230 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-23T00:43:38.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:36.650468+0000 mon.smithi121 (mon.0) 231 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-04-23T00:43:38.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:36.650630+0000 mon.smithi121 (mon.0) 232 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:38.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:36.650966+0000 mon.smithi121 (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:43:38.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:37.651715+0000 mon.smithi121 (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:43:38.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:37.651776+0000 mon.smithi121 (mon.0) 235 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2024-04-23T00:43:38.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:37.652055+0000 mon.smithi121 (mon.0) 236 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:38.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:37.657378+0000 mon.smithi121 (mon.0) 237 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:38.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:37.721166+0000 mgr.smithi121.otezso (mgr.14162) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:39.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:37.651715+0000 mon.smithi121 (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:43:39.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:37.651776+0000 mon.smithi121 (mon.0) 235 : cluster [DBG] osdmap e8: 1 total, 0 up, 1 in 2024-04-23T00:43:39.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:37.652055+0000 mon.smithi121 (mon.0) 236 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:39.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:37.657378+0000 mon.smithi121 (mon.0) 237 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:39.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:37.721166+0000 mgr.smithi121.otezso (mgr.14162) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-23T00:43:40.033 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:36.829492+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:43:40.033 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:36.829578+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:43:40.033 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:38.654654+0000 mon.smithi121 (mon.0) 238 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:40.033 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:38.665686+0000 mon.smithi121 (mon.0) 239 : cluster [INF] osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753] boot 2024-04-23T00:43:40.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:38.665708+0000 mon.smithi121 (mon.0) 240 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-04-23T00:43:40.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:38.665931+0000 mon.smithi121 (mon.0) 241 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:36.829492+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:36.829578+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:38.654654+0000 mon.smithi121 (mon.0) 238 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:38.665686+0000 mon.smithi121 (mon.0) 239 : cluster [INF] osd.0 [v2:172.21.15.121:6802/283218753,v1:172.21.15.121:6803/283218753] boot 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:38.665708+0000 mon.smithi121 (mon.0) 240 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-04-23T00:43:40.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:38.665931+0000 mon.smithi121 (mon.0) 241 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:43:40.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:39.667718+0000 mon.smithi121 (mon.0) 242 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2024-04-23T00:43:40.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:39.721417+0000 mgr.smithi121.otezso (mgr.14162) 90 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:40.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:40.445191+0000 mon.smithi121 (mon.0) 243 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:43:40.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:40.446543+0000 mon.smithi121 (mon.0) 244 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:43:40.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:40.447064+0000 mon.smithi121 (mon.0) 245 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:41.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:39.667718+0000 mon.smithi121 (mon.0) 242 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2024-04-23T00:43:41.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:39.721417+0000 mgr.smithi121.otezso (mgr.14162) 90 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:41.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:40.445191+0000 mon.smithi121 (mon.0) 243 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:43:41.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:40.446543+0000 mon.smithi121 (mon.0) 244 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:43:41.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:40.447064+0000 mon.smithi121 (mon.0) 245 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:41.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:40.444475+0000 mgr.smithi121.otezso (mgr.14162) 91 : audit [DBG] from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:43:41.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:40.446299+0000 mgr.smithi121.otezso (mgr.14162) 92 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:43:42.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:40.444475+0000 mgr.smithi121.otezso (mgr.14162) 91 : audit [DBG] from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:43:42.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:40.446299+0000 mgr.smithi121.otezso (mgr.14162) 92 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:43:42.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:41.721751+0000 mgr.smithi121.otezso (mgr.14162) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:42.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:42.271151+0000 mon.smithi121 (mon.0) 246 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:43.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:41.721751+0000 mgr.smithi121.otezso (mgr.14162) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:43.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:42.271151+0000 mon.smithi121 (mon.0) 246 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:42.271817+0000 mgr.smithi121.otezso (mgr.14162) 94 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:42.274067+0000 mgr.smithi121.otezso (mgr.14162) 95 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:43.321012+0000 mon.smithi121 (mon.0) 247 : audit [INF] from='client.? 172.21.15.121:0/872344991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7f30496-f30d-4737-a044-0ed8ba3f3a8a"}]: dispatch 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:43.324793+0000 mon.smithi121 (mon.0) 248 : audit [INF] from='client.? 172.21.15.121:0/872344991' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7f30496-f30d-4737-a044-0ed8ba3f3a8a"}]': finished 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:43.324831+0000 mon.smithi121 (mon.0) 249 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-04-23T00:43:44.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:43.324933+0000 mon.smithi121 (mon.0) 250 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:43:44.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:42.271817+0000 mgr.smithi121.otezso (mgr.14162) 94 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T00:43:44.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:42.274067+0000 mgr.smithi121.otezso (mgr.14162) 95 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T00:43:44.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:43.321012+0000 mon.smithi121 (mon.0) 247 : audit [INF] from='client.? 172.21.15.121:0/872344991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7f30496-f30d-4737-a044-0ed8ba3f3a8a"}]: dispatch 2024-04-23T00:43:44.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:43.324793+0000 mon.smithi121 (mon.0) 248 : audit [INF] from='client.? 172.21.15.121:0/872344991' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7f30496-f30d-4737-a044-0ed8ba3f3a8a"}]': finished 2024-04-23T00:43:44.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:43.324831+0000 mon.smithi121 (mon.0) 249 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-04-23T00:43:44.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:43.324933+0000 mon.smithi121 (mon.0) 250 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:43:45.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:43.722020+0000 mgr.smithi121.otezso (mgr.14162) 96 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:45.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:43.735753+0000 mon.smithi121 (mon.0) 251 : audit [DBG] from='client.? 172.21.15.121:0/3472809888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:43:45.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:43.722020+0000 mgr.smithi121.otezso (mgr.14162) 96 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:45.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:43.735753+0000 mon.smithi121 (mon.0) 251 : audit [DBG] from='client.? 172.21.15.121:0/3472809888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:43:46.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:45.722413+0000 mgr.smithi121.otezso (mgr.14162) 97 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:47.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:45.722413+0000 mgr.smithi121.otezso (mgr.14162) 97 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:49.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:47.722821+0000 mgr.smithi121.otezso (mgr.14162) 98 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:49.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:47.722821+0000 mgr.smithi121.otezso (mgr.14162) 98 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:51.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:49.723185+0000 mgr.smithi121.otezso (mgr.14162) 99 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:51.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:49.723185+0000 mgr.smithi121.otezso (mgr.14162) 99 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:51.723446+0000 mgr.smithi121.otezso (mgr.14162) 100 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:51.777154+0000 mon.smithi121 (mon.0) 252 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:51.777612+0000 mgr.smithi121.otezso (mgr.14162) 101 : cephadm [INF] Reconfiguring crash.smithi153 (monmap changed)... 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:51.777839+0000 mon.smithi121 (mon.0) 253 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:51.778394+0000 mon.smithi121 (mon.0) 254 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:53.047 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:51.778768+0000 mgr.smithi121.otezso (mgr.14162) 102 : cephadm [INF] Reconfiguring daemon crash.smithi153 on smithi153 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:51.723446+0000 mgr.smithi121.otezso (mgr.14162) 100 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:51.777154+0000 mon.smithi121 (mon.0) 252 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:51.777612+0000 mgr.smithi121.otezso (mgr.14162) 101 : cephadm [INF] Reconfiguring crash.smithi153 (monmap changed)... 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:51.777839+0000 mon.smithi121 (mon.0) 253 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:51.778394+0000 mon.smithi121 (mon.0) 254 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:53.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:51.778768+0000 mgr.smithi121.otezso (mgr.14162) 102 : cephadm [INF] Reconfiguring daemon crash.smithi153 on smithi153 2024-04-23T00:43:54.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.073628+0000 mon.smithi121 (mon.0) 255 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.074284+0000 mon.smithi121 (mon.0) 256 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:53.074758+0000 mgr.smithi121.otezso (mgr.14162) 103 : cephadm [INF] Deploying daemon osd.1 on smithi121 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.094503+0000 mon.smithi121 (mon.0) 257 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:53.095024+0000 mgr.smithi121.otezso (mgr.14162) 104 : cephadm [INF] Reconfiguring mgr.smithi153.olhmgd (monmap changed)... 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.095291+0000 mon.smithi121 (mon.0) 258 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.095953+0000 mon.smithi121 (mon.0) 259 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:53.096448+0000 mon.smithi121 (mon.0) 260 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:53.096888+0000 mgr.smithi121.otezso (mgr.14162) 105 : cephadm [INF] Reconfiguring daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T00:43:54.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:53.723785+0000 mgr.smithi121.otezso (mgr.14162) 106 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:54.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.073628+0000 mon.smithi121 (mon.0) 255 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T00:43:54.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.074284+0000 mon.smithi121 (mon.0) 256 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:53.074758+0000 mgr.smithi121.otezso (mgr.14162) 103 : cephadm [INF] Deploying daemon osd.1 on smithi121 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.094503+0000 mon.smithi121 (mon.0) 257 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:53.095024+0000 mgr.smithi121.otezso (mgr.14162) 104 : cephadm [INF] Reconfiguring mgr.smithi153.olhmgd (monmap changed)... 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.095291+0000 mon.smithi121 (mon.0) 258 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.095953+0000 mon.smithi121 (mon.0) 259 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:53.096448+0000 mon.smithi121 (mon.0) 260 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:53.096888+0000 mgr.smithi121.otezso (mgr.14162) 105 : cephadm [INF] Reconfiguring daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T00:43:54.347 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:53.723785+0000 mgr.smithi121.otezso (mgr.14162) 106 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:55.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:54.497651+0000 mon.smithi121 (mon.0) 261 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:55.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:54.498201+0000 mgr.smithi121.otezso (mgr.14162) 107 : cephadm [INF] Reconfiguring mon.smithi153 (monmap changed)... 2024-04-23T00:43:55.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:54.498433+0000 mon.smithi121 (mon.0) 262 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T00:43:55.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:54.499073+0000 mon.smithi121 (mon.0) 263 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T00:43:55.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:54.499686+0000 mon.smithi121 (mon.0) 264 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:55.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:43:54.500156+0000 mgr.smithi121.otezso (mgr.14162) 108 : cephadm [INF] Reconfiguring daemon mon.smithi153 on smithi153 2024-04-23T00:43:55.816 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:54.497651+0000 mon.smithi121 (mon.0) 261 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:55.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:54.498201+0000 mgr.smithi121.otezso (mgr.14162) 107 : cephadm [INF] Reconfiguring mon.smithi153 (monmap changed)... 2024-04-23T00:43:55.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:54.498433+0000 mon.smithi121 (mon.0) 262 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T00:43:55.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:54.499073+0000 mon.smithi121 (mon.0) 263 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T00:43:55.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:54.499686+0000 mon.smithi121 (mon.0) 264 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:43:55.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:43:54.500156+0000 mgr.smithi121.otezso (mgr.14162) 108 : cephadm [INF] Reconfiguring daemon mon.smithi153 on smithi153 2024-04-23T00:43:56.536 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 1 on host 'smithi121' 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:55.724257+0000 mgr.smithi121.otezso (mgr.14162) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:55.755153+0000 mon.smithi121 (mon.0) 265 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:55.757426+0000 mon.smithi121 (mon.0) 266 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:55.757726+0000 mgr.smithi121.otezso (mgr.14162) 110 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.535501+0000 mon.smithi121 (mon.0) 267 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.540118+0000 mon.smithi121 (mon.0) 268 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.552356+0000 mon.smithi121 (mon.0) 269 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.556373+0000 mon.smithi121 (mon.0) 270 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.558887+0000 mon.smithi121 (mon.0) 271 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.564575+0000 mon.smithi121 (mon.0) 272 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121:3000"}]: dispatch 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.569580+0000 mon.smithi121 (mon.0) 273 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.572922+0000 mon.smithi121 (mon.0) 274 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T00:43:56.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.578101+0000 mon.smithi121 (mon.0) 275 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121:9095"}]: dispatch 2024-04-23T00:43:56.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.581708+0000 mon.smithi121 (mon.0) 276 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:56.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.583746+0000 mon.smithi121 (mon.0) 277 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:55.724257+0000 mgr.smithi121.otezso (mgr.14162) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:55.755153+0000 mon.smithi121 (mon.0) 265 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:55.757426+0000 mon.smithi121 (mon.0) 266 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:55.757726+0000 mgr.smithi121.otezso (mgr.14162) 110 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.535501+0000 mon.smithi121 (mon.0) 267 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.540118+0000 mon.smithi121 (mon.0) 268 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.552356+0000 mon.smithi121 (mon.0) 269 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.556373+0000 mon.smithi121 (mon.0) 270 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.558887+0000 mon.smithi121 (mon.0) 271 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.564575+0000 mon.smithi121 (mon.0) 272 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121:3000"}]: dispatch 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.569580+0000 mon.smithi121 (mon.0) 273 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.572922+0000 mon.smithi121 (mon.0) 274 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.578101+0000 mon.smithi121 (mon.0) 275 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121:9095"}]: dispatch 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.581708+0000 mon.smithi121 (mon.0) 276 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:57.003 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.583746+0000 mon.smithi121 (mon.0) 277 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:43:57.229 DEBUG:teuthology.orchestra.run.smithi121:osd.1> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.1.service 2024-04-23T00:43:57.232 INFO:tasks.cephadm:Deploying osd.2 on smithi121 with /dev/vg_nvme/lv_2... 2024-04-23T00:43:57.232 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-23T00:43:57.264 INFO:journalctl@ceph.osd.1.smithi121.stdout:-- Logs begin at Tue 2024-04-23 00:33:19 UTC. -- 2024-04-23T00:43:57.756 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.552644+0000 mgr.smithi121.otezso (mgr.14162) 111 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2024-04-23T00:43:57.756 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.559185+0000 mgr.smithi121.otezso (mgr.14162) 112 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T00:43:58.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.552644+0000 mgr.smithi121.otezso (mgr.14162) 111 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121:9093"}]: dispatch 2024-04-23T00:43:58.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.559185+0000 mgr.smithi121.otezso (mgr.14162) 112 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T00:43:58.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.564844+0000 mgr.smithi121.otezso (mgr.14162) 113 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121:3000"}]: dispatch 2024-04-23T00:43:58.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.573230+0000 mgr.smithi121.otezso (mgr.14162) 114 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T00:43:58.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:56.578371+0000 mgr.smithi121.otezso (mgr.14162) 115 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121:9095"}]: dispatch 2024-04-23T00:43:58.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.564844+0000 mgr.smithi121.otezso (mgr.14162) 113 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121:3000"}]: dispatch 2024-04-23T00:43:58.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.573230+0000 mgr.smithi121.otezso (mgr.14162) 114 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T00:43:58.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:56.578371+0000 mgr.smithi121.otezso (mgr.14162) 115 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121:9095"}]: dispatch 2024-04-23T00:43:58.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:57.724510+0000 mgr.smithi121.otezso (mgr.14162) 116 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:58.871 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:58.341672+0000 mon.smithi121 (mon.0) 278 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:59.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:57.724510+0000 mgr.smithi121.otezso (mgr.14162) 116 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:43:59.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:58.341672+0000 mon.smithi121 (mon.0) 278 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153384 s, 684 MB/s 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 2024-04-23T00:43:59.722 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-23T00:44:00.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:43:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:59.651692+0000 mon.smithi121 (mon.0) 279 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-23T00:44:00.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:43:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:59.651692+0000 mon.smithi121 (mon.0) 279 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-23T00:44:00.122 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 00:43:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T00:43:59.647+0000 7f5cbfd66080 -1 osd.1 0 log_to_monitors {default=true} 2024-04-23T00:44:00.429 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_2 2024-04-23T00:44:01.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:59.724980+0000 mgr.smithi121.otezso (mgr.14162) 117 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:44:01.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:59.773541+0000 mon.smithi121 (mon.0) 280 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-23T00:44:01.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:43:59.773582+0000 mon.smithi121 (mon.0) 281 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-04-23T00:44:01.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:59.773679+0000 mon.smithi121 (mon.0) 282 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:01.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:43:59.773997+0000 mon.smithi121 (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:01.122 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T00:44:00.774+0000 7f5cb62d7700 -1 osd.1 0 waiting for initial osdmap 2024-04-23T00:44:01.122 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T00:44:00.777+0000 7f5cb2a53700 -1 osd.1 13 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:44:01.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:59.724980+0000 mgr.smithi121.otezso (mgr.14162) 117 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:44:01.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:59.773541+0000 mon.smithi121 (mon.0) 280 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-23T00:44:01.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:43:59.773582+0000 mon.smithi121 (mon.0) 281 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-04-23T00:44:01.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:59.773679+0000 mon.smithi121 (mon.0) 282 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:01.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:43:59.773997+0000 mon.smithi121 (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:02.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:00.770781+0000 mon.smithi121 (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:02.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:00.770838+0000 mon.smithi121 (mon.0) 285 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2024-04-23T00:44:02.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:00.772869+0000 mon.smithi121 (mon.0) 286 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:02.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:00.775692+0000 mon.smithi121 (mon.0) 287 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:02.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:01.775415+0000 mon.smithi121 (mon.0) 288 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:02.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:00.770781+0000 mon.smithi121 (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:02.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:00.770838+0000 mon.smithi121 (mon.0) 285 : cluster [DBG] osdmap e13: 2 total, 1 up, 2 in 2024-04-23T00:44:02.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:00.772869+0000 mon.smithi121 (mon.0) 286 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:02.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:00.775692+0000 mon.smithi121 (mon.0) 287 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:02.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:01.775415+0000 mon.smithi121 (mon.0) 288 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:00.612941+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:00.613044+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:01.725484+0000 mgr.smithi121.otezso (mgr.14162) 118 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:01.779762+0000 mon.smithi121 (mon.0) 289 : cluster [INF] osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702] boot 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:01.779801+0000 mon.smithi121 (mon.0) 290 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-04-23T00:44:03.040 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:01.780259+0000 mon.smithi121 (mon.0) 291 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:03.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:00.612941+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:03.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:00.613044+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:03.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:01.725484+0000 mgr.smithi121.otezso (mgr.14162) 118 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-04-23T00:44:03.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:01.779762+0000 mon.smithi121 (mon.0) 289 : cluster [INF] osd.1 [v2:172.21.15.121:6810/948167702,v1:172.21.15.121:6811/948167702] boot 2024-04-23T00:44:03.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:01.779801+0000 mon.smithi121 (mon.0) 290 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-04-23T00:44:03.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:01.780259+0000 mon.smithi121 (mon.0) 291 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:44:04.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:02.785456+0000 mon.smithi121 (mon.0) 292 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2024-04-23T00:44:04.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:03.701844+0000 mon.smithi121 (mon.0) 293 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:04.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:03.703038+0000 mon.smithi121 (mon.0) 294 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:04.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:03.703584+0000 mon.smithi121 (mon.0) 295 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:04.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:02.785456+0000 mon.smithi121 (mon.0) 292 : cluster [DBG] osdmap e15: 2 total, 2 up, 2 in 2024-04-23T00:44:04.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:03.701844+0000 mon.smithi121 (mon.0) 293 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:04.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:03.703038+0000 mon.smithi121 (mon.0) 294 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:04.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:03.703584+0000 mon.smithi121 (mon.0) 295 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:05.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:03.701099+0000 mgr.smithi121.otezso (mgr.14162) 119 : audit [DBG] from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:05.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:03.702811+0000 mgr.smithi121.otezso (mgr.14162) 120 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:44:05.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:03.725755+0000 mgr.smithi121.otezso (mgr.14162) 121 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:05.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:03.701099+0000 mgr.smithi121.otezso (mgr.14162) 119 : audit [DBG] from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:05.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:03.702811+0000 mgr.smithi121.otezso (mgr.14162) 120 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:44:05.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:03.725755+0000 mgr.smithi121.otezso (mgr.14162) 121 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:06.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:04.950410+0000 mon.smithi121 (mon.0) 296 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:06.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:05.726076+0000 mgr.smithi121.otezso (mgr.14162) 122 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:06.316 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:04.950410+0000 mon.smithi121 (mon.0) 296 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:06.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:05.726076+0000 mgr.smithi121.otezso (mgr.14162) 122 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:07.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:07.077119+0000 mon.smithi121 (mon.0) 297 : audit [INF] from='client.? 172.21.15.121:0/750622805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7974c8fd-b1b3-4053-aeb2-fd363e2866c5"}]: dispatch 2024-04-23T00:44:07.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:07.080761+0000 mon.smithi121 (mon.0) 298 : audit [INF] from='client.? 172.21.15.121:0/750622805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7974c8fd-b1b3-4053-aeb2-fd363e2866c5"}]': finished 2024-04-23T00:44:07.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:07.080797+0000 mon.smithi121 (mon.0) 299 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-04-23T00:44:07.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:07.080894+0000 mon.smithi121 (mon.0) 300 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:07.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:07.077119+0000 mon.smithi121 (mon.0) 297 : audit [INF] from='client.? 172.21.15.121:0/750622805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7974c8fd-b1b3-4053-aeb2-fd363e2866c5"}]: dispatch 2024-04-23T00:44:07.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:07.080761+0000 mon.smithi121 (mon.0) 298 : audit [INF] from='client.? 172.21.15.121:0/750622805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7974c8fd-b1b3-4053-aeb2-fd363e2866c5"}]': finished 2024-04-23T00:44:07.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:07.080797+0000 mon.smithi121 (mon.0) 299 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-04-23T00:44:07.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:07.080894+0000 mon.smithi121 (mon.0) 300 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:08.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:07.484243+0000 mon.smithi121 (mon.0) 301 : audit [DBG] from='client.? 172.21.15.121:0/2146567453' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:08.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:07.726432+0000 mgr.smithi121.otezso (mgr.14162) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:08.621 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:07.484243+0000 mon.smithi121 (mon.0) 301 : audit [DBG] from='client.? 172.21.15.121:0/2146567453' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:08.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:07.726432+0000 mgr.smithi121.otezso (mgr.14162) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:10.969 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:09.726729+0000 mgr.smithi121.otezso (mgr.14162) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:11.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:09.726729+0000 mgr.smithi121.otezso (mgr.14162) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:13.014 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:11.726995+0000 mgr.smithi121.otezso (mgr.14162) 125 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:13.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:11.726995+0000 mgr.smithi121.otezso (mgr.14162) 125 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:14.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:12.958358+0000 mgr.smithi121.otezso (mgr.14162) 126 : cephadm [INF] Detected new or changed devices on smithi121 2024-04-23T00:44:14.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:12.963042+0000 mon.smithi121 (mon.0) 302 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:14.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:13.213296+0000 mon.smithi121 (mon.0) 303 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:14.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:13.215292+0000 mon.smithi121 (mon.0) 304 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:14.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:13.727305+0000 mgr.smithi121.otezso (mgr.14162) 127 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:14.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:12.958358+0000 mgr.smithi121.otezso (mgr.14162) 126 : cephadm [INF] Detected new or changed devices on smithi121 2024-04-23T00:44:14.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:12.963042+0000 mon.smithi121 (mon.0) 302 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:14.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:13.213296+0000 mon.smithi121 (mon.0) 303 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:14.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:13.215292+0000 mon.smithi121 (mon.0) 304 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:14.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:13.727305+0000 mgr.smithi121.otezso (mgr.14162) 127 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:15.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:14.825599+0000 mon.smithi121 (mon.0) 305 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T00:44:15.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:14.826199+0000 mon.smithi121 (mon.0) 306 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:15.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:15.070033+0000 mon.smithi121 (mon.0) 307 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:15.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:14.825599+0000 mon.smithi121 (mon.0) 305 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T00:44:15.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:14.826199+0000 mon.smithi121 (mon.0) 306 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:15.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:15.070033+0000 mon.smithi121 (mon.0) 307 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:16.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:14.826750+0000 mgr.smithi121.otezso (mgr.14162) 128 : cephadm [INF] Deploying daemon osd.2 on smithi121 2024-04-23T00:44:16.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:15.727753+0000 mgr.smithi121.otezso (mgr.14162) 129 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:16.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:14.826750+0000 mgr.smithi121.otezso (mgr.14162) 128 : cephadm [INF] Deploying daemon osd.2 on smithi121 2024-04-23T00:44:16.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:15.727753+0000 mgr.smithi121.otezso (mgr.14162) 129 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:19.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:17.728015+0000 mgr.smithi121.otezso (mgr.14162) 130 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:19.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:17.728015+0000 mgr.smithi121.otezso (mgr.14162) 130 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:20.514 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 2 on host 'smithi121' 2024-04-23T00:44:20.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:19.728447+0000 mgr.smithi121.otezso (mgr.14162) 131 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:20.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:20.257676+0000 mon.smithi121 (mon.0) 308 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:20.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:20.506527+0000 mon.smithi121 (mon.0) 309 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:20.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:20.516456+0000 mon.smithi121 (mon.0) 310 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:21.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:19.728447+0000 mgr.smithi121.otezso (mgr.14162) 131 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:21.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:20.257676+0000 mon.smithi121 (mon.0) 308 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:21.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:20.506527+0000 mon.smithi121 (mon.0) 309 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:21.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:20.516456+0000 mon.smithi121 (mon.0) 310 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:21.318 DEBUG:teuthology.orchestra.run.smithi121:osd.2> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.2.service 2024-04-23T00:44:21.321 INFO:tasks.cephadm:Deploying osd.3 on smithi121 with /dev/vg_nvme/lv_1... 2024-04-23T00:44:21.321 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-23T00:44:21.350 INFO:journalctl@ceph.osd.2.smithi121.stdout:-- Logs begin at Tue 2024-04-23 00:33:19 UTC. -- 2024-04-23T00:44:23.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:21.728852+0000 mgr.smithi121.otezso (mgr.14162) 132 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:23.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:21.728852+0000 mgr.smithi121.otezso (mgr.14162) 132 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:23.952 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 00:44:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T00:44:23.662+0000 7f3e582a4080 -1 osd.2 0 log_to_monitors {default=true} 2024-04-23T00:44:24.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:23.666633+0000 mon.smithi121 (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-23T00:44:24.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:23.729169+0000 mgr.smithi121.otezso (mgr.14162) 133 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:24.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:23.666633+0000 mon.smithi121 (mon.0) 311 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-23T00:44:24.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:23.729169+0000 mgr.smithi121.otezso (mgr.14162) 133 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 10+0 records in 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr:10+0 records out 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.013443 s, 780 MB/s 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr: stderr: 2024-04-23T00:44:24.890 INFO:teuthology.orchestra.run.smithi121.stderr:--> Zapping successful for: 2024-04-23T00:44:25.056 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.051256+0000 mon.smithi121 (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:24.051303+0000 mon.smithi121 (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.051411+0000 mon.smithi121 (mon.0) 314 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.051722+0000 mon.smithi121 (mon.0) 315 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.962747+0000 mon.smithi121 (mon.0) 316 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.965739+0000 mon.smithi121 (mon.0) 317 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:25.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:24.968068+0000 mon.smithi121 (mon.0) 318 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:25.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.051256+0000 mon.smithi121 (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-23T00:44:25.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:24.051303+0000 mon.smithi121 (mon.0) 313 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-04-23T00:44:25.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.051411+0000 mon.smithi121 (mon.0) 314 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:25.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.051722+0000 mon.smithi121 (mon.0) 315 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:25.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.962747+0000 mon.smithi121 (mon.0) 316 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:25.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.965739+0000 mon.smithi121 (mon.0) 317 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:25.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:24.968068+0000 mon.smithi121 (mon.0) 318 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:25.372 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T00:44:25.058+0000 7f3e4e815700 -1 osd.2 0 waiting for initial osdmap 2024-04-23T00:44:25.372 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 00:44:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T00:44:25.062+0000 7f3e48f8d700 -1 osd.2 18 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:44:25.643 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi121:vg_nvme/lv_1 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:25.056945+0000 mon.smithi121 (mon.0) 319 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:25.057021+0000 mon.smithi121 (mon.0) 320 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:25.057364+0000 mon.smithi121 (mon.0) 321 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:25.062768+0000 mon.smithi121 (mon.0) 322 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:25.729457+0000 mgr.smithi121.otezso (mgr.14162) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:25.845448+0000 mon.smithi121 (mon.0) 323 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:44:26.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:25.848928+0000 mon.smithi121 (mon.0) 324 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:25.056945+0000 mon.smithi121 (mon.0) 319 : audit [INF] from='osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:25.057021+0000 mon.smithi121 (mon.0) 320 : cluster [DBG] osdmap e18: 3 total, 2 up, 3 in 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:25.057364+0000 mon.smithi121 (mon.0) 321 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:25.062768+0000 mon.smithi121 (mon.0) 322 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:25.729457+0000 mgr.smithi121.otezso (mgr.14162) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:25.845448+0000 mon.smithi121 (mon.0) 323 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:44:26.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:25.848928+0000 mon.smithi121 (mon.0) 324 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:24.691476+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:24.691649+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:26.059938+0000 mon.smithi121 (mon.0) 325 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:26.063214+0000 mon.smithi121 (mon.0) 326 : cluster [INF] osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417] boot 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:26.063240+0000 mon.smithi121 (mon.0) 327 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:26.063508+0000 mon.smithi121 (mon.0) 328 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:26.063844+0000 mon.smithi121 (mon.0) 329 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:26.786199+0000 mon.smithi121 (mon.0) 330 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:27.540 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:27.067042+0000 mon.smithi121 (mon.0) 331 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-04-23T00:44:27.541 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:27.067086+0000 mon.smithi121 (mon.0) 332 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-04-23T00:44:27.541 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:27.067704+0000 mon.smithi121 (mon.0) 333 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:24.691476+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:24.691649+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:26.059938+0000 mon.smithi121 (mon.0) 325 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:26.063214+0000 mon.smithi121 (mon.0) 326 : cluster [INF] osd.2 [v2:172.21.15.121:6818/4089179417,v1:172.21.15.121:6819/4089179417] boot 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:26.063240+0000 mon.smithi121 (mon.0) 327 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:26.063508+0000 mon.smithi121 (mon.0) 328 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:26.063844+0000 mon.smithi121 (mon.0) 329 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:44:27.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:26.786199+0000 mon.smithi121 (mon.0) 330 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:27.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:27.067042+0000 mon.smithi121 (mon.0) 331 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-04-23T00:44:27.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:27.067086+0000 mon.smithi121 (mon.0) 332 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-04-23T00:44:27.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:27.067704+0000 mon.smithi121 (mon.0) 333 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:27.729847+0000 mgr.smithi121.otezso (mgr.14162) 135 : cluster [DBG] pgmap v100: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:28.065624+0000 mon.smithi121 (mon.0) 334 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:28.065688+0000 mon.smithi121 (mon.0) 335 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:28.187855+0000 mon.smithi121 (mon.0) 336 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:28.189156+0000 mon.smithi121 (mon.0) 337 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:28.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:28.189631+0000 mon.smithi121 (mon.0) 338 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:28.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:27.729847+0000 mgr.smithi121.otezso (mgr.14162) 135 : cluster [DBG] pgmap v100: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:28.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:28.065624+0000 mon.smithi121 (mon.0) 334 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-04-23T00:44:28.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:28.065688+0000 mon.smithi121 (mon.0) 335 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-04-23T00:44:28.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:28.187855+0000 mon.smithi121 (mon.0) 336 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:28.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:28.189156+0000 mon.smithi121 (mon.0) 337 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:28.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:28.189631+0000 mon.smithi121 (mon.0) 338 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:29.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:28.187141+0000 mgr.smithi121.otezso (mgr.14162) 136 : audit [DBG] from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:29.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:28.188963+0000 mgr.smithi121.otezso (mgr.14162) 137 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:44:29.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:29.070776+0000 mon.smithi121 (mon.0) 339 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-04-23T00:44:29.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:28.187141+0000 mgr.smithi121.otezso (mgr.14162) 136 : audit [DBG] from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi121:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:29.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:28.188963+0000 mgr.smithi121.otezso (mgr.14162) 137 : cephadm [INF] Applying service osd.None on host smithi121... 2024-04-23T00:44:29.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:29.070776+0000 mon.smithi121 (mon.0) 339 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-04-23T00:44:30.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:29.730298+0000 mgr.smithi121.otezso (mgr.14162) 138 : cluster [DBG] pgmap v103: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:30.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:29.730298+0000 mgr.smithi121.otezso (mgr.14162) 138 : cluster [DBG] pgmap v103: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:31.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:31.178650+0000 mon.smithi121 (mon.0) 340 : audit [INF] from='client.? 172.21.15.121:0/1447360350' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d2acc81-8c5f-494e-a7d4-20d215a30be5"}]: dispatch 2024-04-23T00:44:31.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:31.182089+0000 mon.smithi121 (mon.0) 341 : audit [INF] from='client.? 172.21.15.121:0/1447360350' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d2acc81-8c5f-494e-a7d4-20d215a30be5"}]': finished 2024-04-23T00:44:31.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:31.182128+0000 mon.smithi121 (mon.0) 342 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-04-23T00:44:31.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:31.182247+0000 mon.smithi121 (mon.0) 343 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:31.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:31.178650+0000 mon.smithi121 (mon.0) 340 : audit [INF] from='client.? 172.21.15.121:0/1447360350' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d2acc81-8c5f-494e-a7d4-20d215a30be5"}]: dispatch 2024-04-23T00:44:31.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:31.182089+0000 mon.smithi121 (mon.0) 341 : audit [INF] from='client.? 172.21.15.121:0/1447360350' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d2acc81-8c5f-494e-a7d4-20d215a30be5"}]': finished 2024-04-23T00:44:31.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:31.182128+0000 mon.smithi121 (mon.0) 342 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-04-23T00:44:31.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:31.182247+0000 mon.smithi121 (mon.0) 343 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:32.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:31.643447+0000 mon.smithi121 (mon.0) 344 : audit [DBG] from='client.? 172.21.15.121:0/2158380222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:32.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:31.730657+0000 mgr.smithi121.otezso (mgr.14162) 139 : cluster [DBG] pgmap v105: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:32.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:31.643447+0000 mon.smithi121 (mon.0) 344 : audit [DBG] from='client.? 172.21.15.121:0/2158380222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:32.622 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:31.730657+0000 mgr.smithi121.otezso (mgr.14162) 139 : cluster [DBG] pgmap v105: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:33.808 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:32.402782+0000 mon.smithi121 (mon.0) 345 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:33.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:32.402782+0000 mon.smithi121 (mon.0) 345 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:34.579 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:33.731000+0000 mgr.smithi121.otezso (mgr.14162) 140 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:34.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:33.731000+0000 mgr.smithi121.otezso (mgr.14162) 140 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:36.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:35.731276+0000 mgr.smithi121.otezso (mgr.14162) 141 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:37.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:35.731276+0000 mgr.smithi121.otezso (mgr.14162) 141 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:38.448 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:37.200420+0000 mgr.smithi121.otezso (mgr.14162) 142 : cephadm [INF] Detected new or changed devices on smithi121 2024-04-23T00:44:38.448 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:37.204891+0000 mon.smithi121 (mon.0) 346 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:38.448 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:37.209702+0000 mon.smithi121 (mon.0) 347 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:38.448 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:37.731597+0000 mgr.smithi121.otezso (mgr.14162) 143 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:38.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:37.200420+0000 mgr.smithi121.otezso (mgr.14162) 142 : cephadm [INF] Detected new or changed devices on smithi121 2024-04-23T00:44:38.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:37.204891+0000 mon.smithi121 (mon.0) 346 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:38.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:37.209702+0000 mon.smithi121 (mon.0) 347 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:38.568 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:37.731597+0000 mgr.smithi121.otezso (mgr.14162) 143 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:39.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:38.471179+0000 mon.smithi121 (mon.0) 348 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T00:44:39.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:38.471740+0000 mon.smithi121 (mon.0) 349 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:39.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:38.472178+0000 mgr.smithi121.otezso (mgr.14162) 144 : cephadm [INF] Deploying daemon osd.3 on smithi121 2024-04-23T00:44:39.582 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:38.471179+0000 mon.smithi121 (mon.0) 348 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T00:44:39.582 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:38.471740+0000 mon.smithi121 (mon.0) 349 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:39.582 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:38.472178+0000 mgr.smithi121.otezso (mgr.14162) 144 : cephadm [INF] Deploying daemon osd.3 on smithi121 2024-04-23T00:44:40.447 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:39.731925+0000 mgr.smithi121.otezso (mgr.14162) 145 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:40.567 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:39.731925+0000 mgr.smithi121.otezso (mgr.14162) 145 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:42.192 INFO:teuthology.orchestra.run.smithi121.stdout:Created osd(s) 3 on host 'smithi121' 2024-04-23T00:44:42.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:41.732386+0000 mgr.smithi121.otezso (mgr.14162) 146 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:42.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:42.186219+0000 mon.smithi121 (mon.0) 350 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:42.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:42.189996+0000 mon.smithi121 (mon.0) 351 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:42.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:42.194674+0000 mon.smithi121 (mon.0) 352 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:42.919 DEBUG:teuthology.orchestra.run.smithi121:osd.3> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.3.service 2024-04-23T00:44:42.921 INFO:tasks.cephadm:Deploying osd.4 on smithi153 with /dev/vg_nvme/lv_4... 2024-04-23T00:44:42.921 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-23T00:44:42.952 INFO:journalctl@ceph.osd.3.smithi121.stdout:-- Logs begin at Tue 2024-04-23 00:33:19 UTC. -- 2024-04-23T00:44:43.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:41.732386+0000 mgr.smithi121.otezso (mgr.14162) 146 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:43.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:42.186219+0000 mon.smithi121 (mon.0) 350 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:43.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:42.189996+0000 mon.smithi121 (mon.0) 351 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:43.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:42.194674+0000 mon.smithi121 (mon.0) 352 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:44.207 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:43.732728+0000 mgr.smithi121.otezso (mgr.14162) 147 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:44.317 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:43.732728+0000 mgr.smithi121.otezso (mgr.14162) 147 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:44.918 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-23T00:44:44.919 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-23T00:44:44.919 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2024-04-23T00:44:44.919 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2024-04-23T00:44:44.919 INFO:teuthology.orchestra.run.smithi153.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112474 s, 932 MB/s 2024-04-23T00:44:44.919 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2024-04-23T00:44:45.488 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi153:vg_nvme/lv_4 2024-04-23T00:44:45.558 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:45.323962+0000 mon.smithi121 (mon.0) 353 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-23T00:44:45.558 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 00:44:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T00:44:45.321+0000 7fe8d1690080 -1 osd.3 0 log_to_monitors {default=true} 2024-04-23T00:44:45.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:45.323962+0000 mon.smithi121 (mon.0) 353 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-23T00:44:46.584 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T00:44:46.386+0000 7fe8c7c01700 -1 osd.3 0 waiting for initial osdmap 2024-04-23T00:44:46.584 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T00:44:46.390+0000 7fe8c0375700 -1 osd.3 25 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:44:46.584 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:45.377972+0000 mon.smithi121 (mon.0) 354 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-23T00:44:46.584 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:45.378031+0000 mon.smithi121 (mon.0) 355 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-04-23T00:44:46.584 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:45.378341+0000 mon.smithi121 (mon.0) 356 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:46.584 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:45.378652+0000 mon.smithi121 (mon.0) 357 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:46.584 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:45.733097+0000 mgr.smithi121.otezso (mgr.14162) 148 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:46.793 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:45.377972+0000 mon.smithi121 (mon.0) 354 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-23T00:44:46.793 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:45.378031+0000 mon.smithi121 (mon.0) 355 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-04-23T00:44:46.793 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:45.378341+0000 mon.smithi121 (mon.0) 356 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:46.793 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:45.378652+0000 mon.smithi121 (mon.0) 357 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T00:44:46.794 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:45.733097+0000 mgr.smithi121.otezso (mgr.14162) 148 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:47.594 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:46.379521+0000 mon.smithi121 (mon.0) 358 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:47.594 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:46.379574+0000 mon.smithi121 (mon.0) 359 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-04-23T00:44:47.594 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:46.379686+0000 mon.smithi121 (mon.0) 360 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:47.594 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:46.387023+0000 mon.smithi121 (mon.0) 361 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:47.595 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:46.932119+0000 mon.smithi121 (mon.0) 362 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:47.799 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:46.379521+0000 mon.smithi121 (mon.0) 358 : audit [INF] from='osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]': finished 2024-04-23T00:44:47.799 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:46.379574+0000 mon.smithi121 (mon.0) 359 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-04-23T00:44:47.799 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:46.379686+0000 mon.smithi121 (mon.0) 360 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:47.800 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:46.387023+0000 mon.smithi121 (mon.0) 361 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:47.800 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:46.932119+0000 mon.smithi121 (mon.0) 362 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:48.553 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:46.327764+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:48.553 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:46.327951+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:48.553 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.388412+0000 mon.smithi121 (mon.0) 363 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:47.390574+0000 mon.smithi121 (mon.0) 364 : cluster [INF] osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544] boot 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:47.390597+0000 mon.smithi121 (mon.0) 365 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.390696+0000 mon.smithi121 (mon.0) 366 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.439316+0000 mgr.smithi121.otezso (mgr.14162) 149 : audit [DBG] from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.439940+0000 mon.smithi121 (mon.0) 367 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:47.441123+0000 mgr.smithi121.otezso (mgr.14162) 150 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.441325+0000 mon.smithi121 (mon.0) 368 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:47.441834+0000 mon.smithi121 (mon.0) 369 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:48.554 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:47.733552+0000 mgr.smithi121.otezso (mgr.14162) 151 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:46.327764+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:46.327951+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.388412+0000 mon.smithi121 (mon.0) 363 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:47.390574+0000 mon.smithi121 (mon.0) 364 : cluster [INF] osd.3 [v2:172.21.15.121:6826/3336760544,v1:172.21.15.121:6827/3336760544] boot 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:47.390597+0000 mon.smithi121 (mon.0) 365 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.390696+0000 mon.smithi121 (mon.0) 366 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.439316+0000 mgr.smithi121.otezso (mgr.14162) 149 : audit [DBG] from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.439940+0000 mon.smithi121 (mon.0) 367 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:47.441123+0000 mgr.smithi121.otezso (mgr.14162) 150 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:44:48.677 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.441325+0000 mon.smithi121 (mon.0) 368 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:44:48.678 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:47.441834+0000 mon.smithi121 (mon.0) 369 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:48.678 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:47.733552+0000 mgr.smithi121.otezso (mgr.14162) 151 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-23T00:44:50.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:49.051855+0000 mon.smithi121 (mon.0) 370 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-04-23T00:44:50.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:49.669350+0000 mon.smithi153 (mon.1) 2 : audit [INF] from='client.? 172.21.15.153:0/4020902575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]: dispatch 2024-04-23T00:44:50.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:49.669733+0000 mon.smithi121 (mon.0) 371 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]: dispatch 2024-04-23T00:44:50.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:49.673233+0000 mon.smithi121 (mon.0) 372 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]': finished 2024-04-23T00:44:50.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:49.673278+0000 mon.smithi121 (mon.0) 373 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-04-23T00:44:50.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:49.673376+0000 mon.smithi121 (mon.0) 374 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:44:50.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:49.733953+0000 mgr.smithi121.otezso (mgr.14162) 152 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:49.051855+0000 mon.smithi121 (mon.0) 370 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:49.669350+0000 mon.smithi153 (mon.1) 2 : audit [INF] from='client.? 172.21.15.153:0/4020902575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]: dispatch 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:49.669733+0000 mon.smithi121 (mon.0) 371 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]: dispatch 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:49.673233+0000 mon.smithi121 (mon.0) 372 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e"}]': finished 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:49.673278+0000 mon.smithi121 (mon.0) 373 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:49.673376+0000 mon.smithi121 (mon.0) 374 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:44:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:49.733953+0000 mgr.smithi121.otezso (mgr.14162) 152 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:51.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:50.067959+0000 mon.smithi153 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.153:0/31580668' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:51.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:50.369938+0000 mon.smithi121 (mon.0) 375 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:51.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:50.375407+0000 mon.smithi121 (mon.0) 376 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:51.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:50.377884+0000 mon.smithi121 (mon.0) 377 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:51.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:50.067959+0000 mon.smithi153 (mon.1) 3 : audit [DBG] from='client.? 172.21.15.153:0/31580668' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:44:51.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:50.369938+0000 mon.smithi121 (mon.0) 375 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:51.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:50.375407+0000 mon.smithi121 (mon.0) 376 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:51.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:50.377884+0000 mon.smithi121 (mon.0) 377 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:52.308 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:51.734485+0000 mgr.smithi121.otezso (mgr.14162) 153 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:52.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:51.734485+0000 mgr.smithi121.otezso (mgr.14162) 153 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:54.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:53.734838+0000 mgr.smithi121.otezso (mgr.14162) 154 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:54.281 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:53.734838+0000 mgr.smithi121.otezso (mgr.14162) 154 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:55.945 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:54.694868+0000 mon.smithi121 (mon.0) 378 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:55.945 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:54.699930+0000 mon.smithi121 (mon.0) 379 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:55.945 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:55.462005+0000 mon.smithi121 (mon.0) 380 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T00:44:55.945 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:55.462868+0000 mon.smithi121 (mon.0) 381 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:56.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:54.694868+0000 mon.smithi121 (mon.0) 378 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:56.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:54.699930+0000 mon.smithi121 (mon.0) 379 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:56.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:55.462005+0000 mon.smithi121 (mon.0) 380 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T00:44:56.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:55.462868+0000 mon.smithi121 (mon.0) 381 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:44:56.753 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:44:55.463493+0000 mgr.smithi121.otezso (mgr.14162) 155 : cephadm [INF] Deploying daemon osd.4 on smithi153 2024-04-23T00:44:56.753 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:55.735322+0000 mgr.smithi121.otezso (mgr.14162) 156 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:57.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:44:55.463493+0000 mgr.smithi121.otezso (mgr.14162) 155 : cephadm [INF] Deploying daemon osd.4 on smithi153 2024-04-23T00:44:57.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:55.735322+0000 mgr.smithi121.otezso (mgr.14162) 156 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:58.876 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:57.735738+0000 mgr.smithi121.otezso (mgr.14162) 157 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:58.927 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 4 on host 'smithi153' 2024-04-23T00:44:59.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:57.735738+0000 mgr.smithi121.otezso (mgr.14162) 157 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:44:59.627 DEBUG:teuthology.orchestra.run.smithi153:osd.4> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.4.service 2024-04-23T00:44:59.629 INFO:tasks.cephadm:Deploying osd.5 on smithi153 with /dev/vg_nvme/lv_3... 2024-04-23T00:44:59.629 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-23T00:44:59.660 INFO:journalctl@ceph.osd.4.smithi153.stdout:-- Logs begin at Tue 2024-04-23 00:33:27 UTC. -- 2024-04-23T00:44:59.887 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:58.920373+0000 mon.smithi121 (mon.0) 382 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:44:59.887 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:58.923082+0000 mon.smithi121 (mon.0) 383 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:44:59.887 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:44:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:44:58.929502+0000 mon.smithi121 (mon.0) 384 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:00.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:58.920373+0000 mon.smithi121 (mon.0) 382 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:00.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:58.923082+0000 mon.smithi121 (mon.0) 383 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:00.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:44:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:44:58.929502+0000 mon.smithi121 (mon.0) 384 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:01.043 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:44:59.736263+0000 mgr.smithi121.otezso (mgr.14162) 158 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:01.054 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:44:59.736263+0000 mgr.smithi121.otezso (mgr.14162) 158 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129194 s, 812 MB/s 2024-04-23T00:45:02.155 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2024-04-23T00:45:02.317 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 00:45:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T00:45:02.025+0000 7f80b40ab080 -1 osd.4 0 log_to_monitors {default=true} 2024-04-23T00:45:03.014 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:01.736587+0000 mgr.smithi121.otezso (mgr.14162) 159 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:03.015 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.029989+0000 mon.smithi153 (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T00:45:03.015 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.030343+0000 mon.smithi121 (mon.0) 385 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T00:45:03.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:01.736587+0000 mgr.smithi121.otezso (mgr.14162) 159 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:03.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.029989+0000 mon.smithi153 (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T00:45:03.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.030343+0000 mon.smithi121 (mon.0) 385 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T00:45:03.330 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi153:vg_nvme/lv_3 2024-04-23T00:45:03.973 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T00:45:03.807+0000 7f80aa61c700 -1 osd.4 0 waiting for initial osdmap 2024-04-23T00:45:03.974 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T00:45:03.811+0000 7f80a4d94700 -1 osd.4 30 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.799559+0000 mon.smithi121 (mon.0) 386 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:02.799607+0000 mon.smithi121 (mon.0) 387 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.799707+0000 mon.smithi121 (mon.0) 388 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.800179+0000 mon.smithi153 (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.800389+0000 mon.smithi121 (mon.0) 389 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:02.932072+0000 mon.smithi121 (mon.0) 390 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:03.974 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:03.218818+0000 mon.smithi121 (mon.0) 391 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:04.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.799559+0000 mon.smithi121 (mon.0) 386 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-23T00:45:04.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:02.799607+0000 mon.smithi121 (mon.0) 387 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-04-23T00:45:04.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.799707+0000 mon.smithi121 (mon.0) 388 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:04.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.800179+0000 mon.smithi153 (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:04.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.800389+0000 mon.smithi121 (mon.0) 389 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:04.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:02.932072+0000 mon.smithi121 (mon.0) 390 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:04.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:03.218818+0000 mon.smithi121 (mon.0) 391 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:03.736889+0000 mgr.smithi121.otezso (mgr.14162) 160 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:03.802335+0000 mon.smithi121 (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:03.802474+0000 mon.smithi121 (mon.0) 393 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:03.802929+0000 mon.smithi121 (mon.0) 394 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:03.807877+0000 mon.smithi121 (mon.0) 395 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:04.803957+0000 mon.smithi121 (mon.0) 396 : cluster [INF] osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181] boot 2024-04-23T00:45:05.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:04.803998+0000 mon.smithi121 (mon.0) 397 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-04-23T00:45:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:04.804151+0000 mon.smithi121 (mon.0) 398 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:05.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:03.736889+0000 mgr.smithi121.otezso (mgr.14162) 160 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:03.802335+0000 mon.smithi121 (mon.0) 392 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:03.802474+0000 mon.smithi121 (mon.0) 393 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:03.802929+0000 mon.smithi121 (mon.0) 394 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:03.807877+0000 mon.smithi121 (mon.0) 395 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:04.803957+0000 mon.smithi121 (mon.0) 396 : cluster [INF] osd.4 [v2:172.21.15.153:6800/1662652181,v1:172.21.15.153:6801/1662652181] boot 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:04.803998+0000 mon.smithi121 (mon.0) 397 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-04-23T00:45:05.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:04.804151+0000 mon.smithi121 (mon.0) 398 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:45:06.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:03.065984+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:06.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:03.066079+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:06.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:05.737377+0000 mgr.smithi121.otezso (mgr.14162) 161 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:06.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:05.805504+0000 mon.smithi121 (mon.0) 399 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-04-23T00:45:06.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:03.065984+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:06.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:03.066079+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:06.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:05.737377+0000 mgr.smithi121.otezso (mgr.14162) 161 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:06.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:05.805504+0000 mon.smithi121 (mon.0) 399 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:06.807345+0000 mon.smithi121 (mon.0) 400 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:06.892694+0000 mgr.smithi121.otezso (mgr.14162) 162 : audit [DBG] from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:06.893610+0000 mon.smithi121 (mon.0) 401 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:06.894824+0000 mgr.smithi121.otezso (mgr.14162) 163 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:06.895061+0000 mon.smithi121 (mon.0) 402 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:06.895718+0000 mon.smithi121 (mon.0) 403 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:07.297466+0000 mgr.smithi121.otezso (mgr.14162) 164 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:07.302134+0000 mon.smithi121 (mon.0) 404 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:07.543962+0000 mon.smithi121 (mon.0) 405 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:08.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:07.546615+0000 mon.smithi121 (mon.0) 406 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:08.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:07.737702+0000 mgr.smithi121.otezso (mgr.14162) 165 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:08.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:06.807345+0000 mon.smithi121 (mon.0) 400 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:06.892694+0000 mgr.smithi121.otezso (mgr.14162) 162 : audit [DBG] from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:06.893610+0000 mon.smithi121 (mon.0) 401 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:06.894824+0000 mgr.smithi121.otezso (mgr.14162) 163 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:06.895061+0000 mon.smithi121 (mon.0) 402 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:06.895718+0000 mon.smithi121 (mon.0) 403 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:07.297466+0000 mgr.smithi121.otezso (mgr.14162) 164 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:07.302134+0000 mon.smithi121 (mon.0) 404 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:07.543962+0000 mon.smithi121 (mon.0) 405 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:07.546615+0000 mon.smithi121 (mon.0) 406 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:08.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:07.737702+0000 mgr.smithi121.otezso (mgr.14162) 165 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:09.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:09.181807+0000 mon.smithi153 (mon.1) 6 : audit [INF] from='client.? 172.21.15.153:0/1154116159' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]: dispatch 2024-04-23T00:45:09.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:09.182434+0000 mon.smithi121 (mon.0) 407 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]: dispatch 2024-04-23T00:45:09.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:09.185819+0000 mon.smithi121 (mon.0) 408 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]': finished 2024-04-23T00:45:09.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:09.185864+0000 mon.smithi121 (mon.0) 409 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-04-23T00:45:09.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:09.185946+0000 mon.smithi121 (mon.0) 410 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:09.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:09.181807+0000 mon.smithi153 (mon.1) 6 : audit [INF] from='client.? 172.21.15.153:0/1154116159' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]: dispatch 2024-04-23T00:45:09.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:09.182434+0000 mon.smithi121 (mon.0) 407 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]: dispatch 2024-04-23T00:45:09.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:09.185819+0000 mon.smithi121 (mon.0) 408 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2506530-71bf-42e3-9691-1e05ff732924"}]': finished 2024-04-23T00:45:09.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:09.185864+0000 mon.smithi121 (mon.0) 409 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-04-23T00:45:09.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:09.185946+0000 mon.smithi121 (mon.0) 410 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:10.677 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:09.610706+0000 mon.smithi153 (mon.1) 7 : audit [DBG] from='client.? 172.21.15.153:0/1574675472' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:10.677 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:09.738046+0000 mgr.smithi121.otezso (mgr.14162) 166 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:10.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:09.610706+0000 mon.smithi153 (mon.1) 7 : audit [DBG] from='client.? 172.21.15.153:0/1574675472' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:10.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:09.738046+0000 mgr.smithi121.otezso (mgr.14162) 166 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:12.986 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:11.738394+0000 mgr.smithi121.otezso (mgr.14162) 167 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:12.987 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:11.829069+0000 mon.smithi121 (mon.0) 411 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:12.987 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:11.834387+0000 mon.smithi121 (mon.0) 412 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:13.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:11.738394+0000 mgr.smithi121.otezso (mgr.14162) 167 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:13.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:11.829069+0000 mon.smithi121 (mon.0) 411 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:13.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:11.834387+0000 mon.smithi121 (mon.0) 412 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:14.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:13.738926+0000 mgr.smithi121.otezso (mgr.14162) 168 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:14.372 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:13.738926+0000 mgr.smithi121.otezso (mgr.14162) 168 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:16.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:15.625307+0000 mon.smithi121 (mon.0) 413 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T00:45:16.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:15.626139+0000 mon.smithi121 (mon.0) 414 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:16.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:15.625307+0000 mon.smithi121 (mon.0) 413 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T00:45:16.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:15.626139+0000 mon.smithi121 (mon.0) 414 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:16.975 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:15.626800+0000 mgr.smithi121.otezso (mgr.14162) 169 : cephadm [INF] Deploying daemon osd.5 on smithi153 2024-04-23T00:45:16.975 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:15.739342+0000 mgr.smithi121.otezso (mgr.14162) 170 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:17.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:15.626800+0000 mgr.smithi121.otezso (mgr.14162) 169 : cephadm [INF] Deploying daemon osd.5 on smithi153 2024-04-23T00:45:17.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:15.739342+0000 mgr.smithi121.otezso (mgr.14162) 170 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:19.048 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:17.739882+0000 mgr.smithi121.otezso (mgr.14162) 171 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:19.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:17.739882+0000 mgr.smithi121.otezso (mgr.14162) 171 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:19.186 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 5 on host 'smithi153' 2024-04-23T00:45:19.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:19.174020+0000 mon.smithi121 (mon.0) 415 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:19.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:19.180232+0000 mon.smithi121 (mon.0) 416 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:19.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:19.187509+0000 mon.smithi121 (mon.0) 417 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:19.898 DEBUG:teuthology.orchestra.run.smithi153:osd.5> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.5.service 2024-04-23T00:45:19.900 INFO:tasks.cephadm:Deploying osd.6 on smithi153 with /dev/vg_nvme/lv_2... 2024-04-23T00:45:19.900 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-23T00:45:19.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:19.174020+0000 mon.smithi121 (mon.0) 415 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:19.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:19.180232+0000 mon.smithi121 (mon.0) 416 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:19.915 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:19.187509+0000 mon.smithi121 (mon.0) 417 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:19.930 INFO:journalctl@ceph.osd.5.smithi153.stdout:-- Logs begin at Tue 2024-04-23 00:33:27 UTC. -- 2024-04-23T00:45:21.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:19.740205+0000 mgr.smithi121.otezso (mgr.14162) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:21.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:19.740205+0000 mgr.smithi121.otezso (mgr.14162) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:22.568 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 00:45:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T00:45:22.277+0000 7f3461a87080 -1 osd.5 0 log_to_monitors {default=true} 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0122005 s, 859 MB/s 2024-04-23T00:45:22.687 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2024-04-23T00:45:22.977 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:21.740479+0000 mgr.smithi121.otezso (mgr.14162) 173 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:22.978 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.281087+0000 mon.smithi153 (mon.1) 8 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T00:45:22.978 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.281389+0000 mon.smithi121 (mon.0) 418 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T00:45:23.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:21.740479+0000 mgr.smithi121.otezso (mgr.14162) 173 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:23.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.281087+0000 mon.smithi153 (mon.1) 8 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T00:45:23.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.281389+0000 mon.smithi121 (mon.0) 418 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T00:45:23.469 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi153:vg_nvme/lv_2 2024-04-23T00:45:24.069 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T00:45:23.811+0000 7f3457ff8700 -1 osd.5 0 waiting for initial osdmap 2024-04-23T00:45:24.069 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T00:45:23.816+0000 7f3452770700 -1 osd.5 36 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.808693+0000 mon.smithi121 (mon.0) 419 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:22.808761+0000 mon.smithi121 (mon.0) 420 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.808867+0000 mon.smithi121 (mon.0) 421 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.809413+0000 mon.smithi153 (mon.1) 9 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:22.809643+0000 mon.smithi121 (mon.0) 422 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:23.333269+0000 mon.smithi121 (mon.0) 423 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:24.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:23.543877+0000 mon.smithi121 (mon.0) 424 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:24.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:23.740914+0000 mgr.smithi121.otezso (mgr.14162) 174 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.808693+0000 mon.smithi121 (mon.0) 419 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:22.808761+0000 mon.smithi121 (mon.0) 420 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.808867+0000 mon.smithi121 (mon.0) 421 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.809413+0000 mon.smithi153 (mon.1) 9 : audit [INF] from='osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:22.809643+0000 mon.smithi121 (mon.0) 422 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:23.333269+0000 mon.smithi121 (mon.0) 423 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:23.543877+0000 mon.smithi121 (mon.0) 424 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:24.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:23.740914+0000 mgr.smithi121.otezso (mgr.14162) 174 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:25.008 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:23.808196+0000 mon.smithi121 (mon.0) 425 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:25.008 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:23.808279+0000 mon.smithi121 (mon.0) 426 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-04-23T00:45:25.008 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:23.808595+0000 mon.smithi121 (mon.0) 427 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:25.008 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:23.812320+0000 mon.smithi121 (mon.0) 428 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:25.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:23.808196+0000 mon.smithi121 (mon.0) 425 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:25.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:23.808279+0000 mon.smithi121 (mon.0) 426 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-04-23T00:45:25.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:23.808595+0000 mon.smithi121 (mon.0) 427 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:25.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:23.812320+0000 mon.smithi121 (mon.0) 428 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:26.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:23.292686+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:26.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:23.292771+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:26.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:24.813340+0000 mon.smithi121 (mon.0) 429 : cluster [INF] osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392] boot 2024-04-23T00:45:26.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:24.813407+0000 mon.smithi121 (mon.0) 430 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-04-23T00:45:26.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:24.813909+0000 mon.smithi121 (mon.0) 431 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:26.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:25.741440+0000 mgr.smithi121.otezso (mgr.14162) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:23.292686+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:23.292771+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:24.813340+0000 mon.smithi121 (mon.0) 429 : cluster [INF] osd.5 [v2:172.21.15.153:6808/3077775392,v1:172.21.15.153:6809/3077775392] boot 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:24.813407+0000 mon.smithi121 (mon.0) 430 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:24.813909+0000 mon.smithi121 (mon.0) 431 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:45:26.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:25.741440+0000 mgr.smithi121.otezso (mgr.14162) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-04-23T00:45:27.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:25.822177+0000 mon.smithi121 (mon.0) 432 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-04-23T00:45:27.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:25.852819+0000 mon.smithi121 (mon.0) 433 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:25.853004+0000 mon.smithi121 (mon.0) 434 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:26.166032+0000 mgr.smithi121.otezso (mgr.14162) 176 : audit [DBG] from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:26.166971+0000 mon.smithi121 (mon.0) 435 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:26.168351+0000 mgr.smithi121.otezso (mgr.14162) 177 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:26.168621+0000 mon.smithi121 (mon.0) 436 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:27.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:26.169277+0000 mon.smithi121 (mon.0) 437 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:25.822177+0000 mon.smithi121 (mon.0) 432 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:25.852819+0000 mon.smithi121 (mon.0) 433 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:25.853004+0000 mon.smithi121 (mon.0) 434 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:26.166032+0000 mgr.smithi121.otezso (mgr.14162) 176 : audit [DBG] from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:26.166971+0000 mon.smithi121 (mon.0) 435 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:26.168351+0000 mgr.smithi121.otezso (mgr.14162) 177 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:27.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:26.168621+0000 mon.smithi121 (mon.0) 436 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:27.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:26.169277+0000 mon.smithi121 (mon.0) 437 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:26.826011+0000 mon.smithi121 (mon.0) 438 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-04-23T00:45:28.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:27.741856+0000 mgr.smithi121.otezso (mgr.14162) 178 : cluster [DBG] pgmap v149: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:28.133 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:26.826011+0000 mon.smithi121 (mon.0) 438 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-04-23T00:45:28.133 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:27.741856+0000 mgr.smithi121.otezso (mgr.14162) 178 : cluster [DBG] pgmap v149: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:29.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:28.351715+0000 mgr.smithi121.otezso (mgr.14162) 179 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:29.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:28.356587+0000 mon.smithi121 (mon.0) 439 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:29.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:28.361250+0000 mon.smithi121 (mon.0) 440 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:29.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:28.363999+0000 mon.smithi121 (mon.0) 441 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:29.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:28.351715+0000 mgr.smithi121.otezso (mgr.14162) 179 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:29.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:28.356587+0000 mon.smithi121 (mon.0) 439 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:29.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:28.361250+0000 mon.smithi121 (mon.0) 440 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:29.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:28.363999+0000 mon.smithi121 (mon.0) 441 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:30.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:29.643795+0000 mon.smithi153 (mon.1) 10 : audit [INF] from='client.? 172.21.15.153:0/486217217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]: dispatch 2024-04-23T00:45:30.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:29.644249+0000 mon.smithi121 (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]: dispatch 2024-04-23T00:45:30.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:29.649244+0000 mon.smithi121 (mon.0) 443 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]': finished 2024-04-23T00:45:30.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:29.649317+0000 mon.smithi121 (mon.0) 444 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-04-23T00:45:30.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:29.649417+0000 mon.smithi121 (mon.0) 445 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:30.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:29.742267+0000 mgr.smithi121.otezso (mgr.14162) 180 : cluster [DBG] pgmap v151: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:30.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:30.050906+0000 mon.smithi153 (mon.1) 11 : audit [DBG] from='client.? 172.21.15.153:0/153565528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:30.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:29.643795+0000 mon.smithi153 (mon.1) 10 : audit [INF] from='client.? 172.21.15.153:0/486217217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]: dispatch 2024-04-23T00:45:30.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:29.644249+0000 mon.smithi121 (mon.0) 442 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]: dispatch 2024-04-23T00:45:30.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:29.649244+0000 mon.smithi121 (mon.0) 443 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de0de950-d455-48e7-8be6-670f8fead757"}]': finished 2024-04-23T00:45:30.818 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:29.649317+0000 mon.smithi121 (mon.0) 444 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-04-23T00:45:30.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:29.649417+0000 mon.smithi121 (mon.0) 445 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:30.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:29.742267+0000 mgr.smithi121.otezso (mgr.14162) 180 : cluster [DBG] pgmap v151: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:30.819 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:30.050906+0000 mon.smithi153 (mon.1) 11 : audit [DBG] from='client.? 172.21.15.153:0/153565528' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:32.938 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:31.742559+0000 mgr.smithi121.otezso (mgr.14162) 181 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:32.938 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:32.616973+0000 mon.smithi121 (mon.0) 446 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:32.938 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:32.619807+0000 mon.smithi121 (mon.0) 447 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:33.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:31.742559+0000 mgr.smithi121.otezso (mgr.14162) 181 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:33.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:32.616973+0000 mon.smithi121 (mon.0) 446 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:33.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:32.619807+0000 mon.smithi121 (mon.0) 447 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:34.308 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:33.742841+0000 mgr.smithi121.otezso (mgr.14162) 182 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:34.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:33.742841+0000 mgr.smithi121.otezso (mgr.14162) 182 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:36.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:35.692046+0000 mon.smithi121 (mon.0) 448 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T00:45:36.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:35.692867+0000 mon.smithi121 (mon.0) 449 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:36.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:35.692046+0000 mon.smithi121 (mon.0) 448 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T00:45:36.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:35.692867+0000 mon.smithi121 (mon.0) 449 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:37.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:35.693503+0000 mgr.smithi121.otezso (mgr.14162) 183 : cephadm [INF] Deploying daemon osd.6 on smithi153 2024-04-23T00:45:37.002 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:35.743174+0000 mgr.smithi121.otezso (mgr.14162) 184 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:37.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:35.693503+0000 mgr.smithi121.otezso (mgr.14162) 183 : cephadm [INF] Deploying daemon osd.6 on smithi153 2024-04-23T00:45:37.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:35.743174+0000 mgr.smithi121.otezso (mgr.14162) 184 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:39.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:37.743637+0000 mgr.smithi121.otezso (mgr.14162) 185 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:39.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:37.743637+0000 mgr.smithi121.otezso (mgr.14162) 185 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:39.214 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 6 on host 'smithi153' 2024-04-23T00:45:39.918 DEBUG:teuthology.orchestra.run.smithi153:osd.6> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.6.service 2024-04-23T00:45:39.920 INFO:tasks.cephadm:Deploying osd.7 on smithi153 with /dev/vg_nvme/lv_1... 2024-04-23T00:45:39.921 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-23T00:45:39.946 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:39.202775+0000 mon.smithi121 (mon.0) 450 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:39.946 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:39.208242+0000 mon.smithi121 (mon.0) 451 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:39.946 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:39.215778+0000 mon.smithi121 (mon.0) 452 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:39.952 INFO:journalctl@ceph.osd.6.smithi153.stdout:-- Logs begin at Tue 2024-04-23 00:33:27 UTC. -- 2024-04-23T00:45:40.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:39.202775+0000 mon.smithi121 (mon.0) 450 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:40.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:39.208242+0000 mon.smithi121 (mon.0) 451 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:40.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:39.215778+0000 mon.smithi121 (mon.0) 452 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:41.048 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:39.743924+0000 mgr.smithi121.otezso (mgr.14162) 186 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:41.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:39.743924+0000 mgr.smithi121.otezso (mgr.14162) 186 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:42.012 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:41.744196+0000 mgr.smithi121.otezso (mgr.14162) 187 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:42.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:41.744196+0000 mgr.smithi121.otezso (mgr.14162) 187 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:42.568 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 00:45:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T00:45:42.284+0000 7fad877fc080 -1 osd.6 0 log_to_monitors {default=true} 2024-04-23T00:45:42.575 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-23T00:45:42.575 INFO:teuthology.orchestra.run.smithi153.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-23T00:45:42.575 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10+0 records in 2024-04-23T00:45:42.575 INFO:teuthology.orchestra.run.smithi153.stderr:10+0 records out 2024-04-23T00:45:42.575 INFO:teuthology.orchestra.run.smithi153.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0141274 s, 742 MB/s 2024-04-23T00:45:42.576 INFO:teuthology.orchestra.run.smithi153.stderr:--> Zapping successful for: 2024-04-23T00:45:43.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.289589+0000 mon.smithi153 (mon.1) 12 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T00:45:43.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.289919+0000 mon.smithi121 (mon.0) 453 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T00:45:43.083 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.289589+0000 mon.smithi153 (mon.1) 12 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T00:45:43.083 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.289919+0000 mon.smithi121 (mon.0) 453 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T00:45:43.663 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi153:vg_nvme/lv_1 2024-04-23T00:45:44.069 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T00:45:43.866+0000 7fad7f570700 -1 osd.6 0 waiting for initial osdmap 2024-04-23T00:45:44.069 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T00:45:43.873+0000 7fad784e5700 -1 osd.6 42 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:45:44.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.861805+0000 mon.smithi121 (mon.0) 454 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-23T00:45:44.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:42.861884+0000 mon.smithi121 (mon.0) 455 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-04-23T00:45:44.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.862019+0000 mon.smithi121 (mon.0) 456 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:44.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.867159+0000 mon.smithi153 (mon.1) 13 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:44.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:42.868101+0000 mon.smithi121 (mon.0) 457 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:44.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:43.436485+0000 mon.smithi121 (mon.0) 458 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:44.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:43.737482+0000 mon.smithi121 (mon.0) 459 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:44.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:43.744438+0000 mgr.smithi121.otezso (mgr.14162) 188 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:44.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.861805+0000 mon.smithi121 (mon.0) 454 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-23T00:45:44.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:42.861884+0000 mon.smithi121 (mon.0) 455 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-04-23T00:45:44.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.862019+0000 mon.smithi121 (mon.0) 456 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:44.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.867159+0000 mon.smithi153 (mon.1) 13 : audit [INF] from='osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:44.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:42.868101+0000 mon.smithi121 (mon.0) 457 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:45:44.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:43.436485+0000 mon.smithi121 (mon.0) 458 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:44.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:43.737482+0000 mon.smithi121 (mon.0) 459 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:44.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:43.744438+0000 mgr.smithi121.otezso (mgr.14162) 188 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-04-23T00:45:45.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:43.864128+0000 mon.smithi121 (mon.0) 460 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:45.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:43.864205+0000 mon.smithi121 (mon.0) 461 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2024-04-23T00:45:45.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:43.864514+0000 mon.smithi121 (mon.0) 462 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:45.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:43.868291+0000 mon.smithi121 (mon.0) 463 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:45.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:43.864128+0000 mon.smithi121 (mon.0) 460 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:45:45.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:43.864205+0000 mon.smithi121 (mon.0) 461 : cluster [DBG] osdmap e42: 7 total, 6 up, 7 in 2024-04-23T00:45:45.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:43.864514+0000 mon.smithi121 (mon.0) 462 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:45.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:43.868291+0000 mon.smithi121 (mon.0) 463 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:46.117 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:43.312202+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:46.118 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:43.312291+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:46.118 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:44.865704+0000 mon.smithi121 (mon.0) 464 : cluster [INF] osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165] boot 2024-04-23T00:45:46.118 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:44.865753+0000 mon.smithi121 (mon.0) 465 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-04-23T00:45:46.118 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:44.865903+0000 mon.smithi121 (mon.0) 466 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:46.118 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:45.744839+0000 mgr.smithi121.otezso (mgr.14162) 189 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:46.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:43.312202+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:45:46.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:43.312291+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:45:46.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:44.865704+0000 mon.smithi121 (mon.0) 464 : cluster [INF] osd.6 [v2:172.21.15.153:6816/1407250165,v1:172.21.15.153:6817/1407250165] boot 2024-04-23T00:45:46.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:44.865753+0000 mon.smithi121 (mon.0) 465 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-04-23T00:45:46.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:44.865903+0000 mon.smithi121 (mon.0) 466 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:45:46.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:45.744839+0000 mgr.smithi121.otezso (mgr.14162) 189 : cluster [DBG] pgmap v162: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:47.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:45.867745+0000 mon.smithi121 (mon.0) 467 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-04-23T00:45:47.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:45.867745+0000 mon.smithi121 (mon.0) 467 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:46.877447+0000 mon.smithi121 (mon.0) 468 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:47.274060+0000 mgr.smithi121.otezso (mgr.14162) 190 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:47.275052+0000 mon.smithi121 (mon.0) 469 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:47.276364+0000 mgr.smithi121.otezso (mgr.14162) 191 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:47.276643+0000 mon.smithi121 (mon.0) 470 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:47.277370+0000 mon.smithi121 (mon.0) 471 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:48.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:47.745192+0000 mgr.smithi121.otezso (mgr.14162) 192 : cluster [DBG] pgmap v165: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:48.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:46.877447+0000 mon.smithi121 (mon.0) 468 : cluster [DBG] osdmap e45: 7 total, 7 up, 7 in 2024-04-23T00:45:48.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:47.274060+0000 mgr.smithi121.otezso (mgr.14162) 190 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi153:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:45:48.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:47.275052+0000 mon.smithi121 (mon.0) 469 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-23T00:45:48.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:47.276364+0000 mgr.smithi121.otezso (mgr.14162) 191 : cephadm [INF] Applying service osd.None on host smithi153... 2024-04-23T00:45:48.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:47.276643+0000 mon.smithi121 (mon.0) 470 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-23T00:45:48.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:47.277370+0000 mon.smithi121 (mon.0) 471 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:48.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:47.745192+0000 mgr.smithi121.otezso (mgr.14162) 192 : cluster [DBG] pgmap v165: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:50.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:48.991033+0000 mgr.smithi121.otezso (mgr.14162) 193 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:50.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:48.996125+0000 mon.smithi121 (mon.0) 472 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:48.999485+0000 mon.smithi121 (mon.0) 473 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:49.002564+0000 mon.smithi121 (mon.0) 474 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:50.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:49.006514+0000 mon.smithi121 (mon.0) 475 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:49.745614+0000 mgr.smithi121.otezso (mgr.14162) 194 : cluster [DBG] pgmap v166: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:50.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:48.991033+0000 mgr.smithi121.otezso (mgr.14162) 193 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:45:50.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:48.996125+0000 mon.smithi121 (mon.0) 472 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:48.999485+0000 mon.smithi121 (mon.0) 473 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:49.002564+0000 mon.smithi121 (mon.0) 474 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:45:50.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:49.006514+0000 mon.smithi121 (mon.0) 475 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:45:50.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:49.745614+0000 mgr.smithi121.otezso (mgr.14162) 194 : cluster [DBG] pgmap v166: 1 pgs: 1 peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:51.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:50.245591+0000 mon.smithi153 (mon.1) 14 : audit [INF] from='client.? 172.21.15.153:0/2574161419' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]: dispatch 2024-04-23T00:45:51.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:50.246150+0000 mon.smithi121 (mon.0) 476 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]: dispatch 2024-04-23T00:45:51.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:50.254949+0000 mon.smithi121 (mon.0) 477 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]': finished 2024-04-23T00:45:51.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:50.255005+0000 mon.smithi121 (mon.0) 478 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-04-23T00:45:51.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:50.255173+0000 mon.smithi121 (mon.0) 479 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:45:51.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:50.662959+0000 mon.smithi153 (mon.1) 15 : audit [DBG] from='client.? 172.21.15.153:0/3027772601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:51.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:50.245591+0000 mon.smithi153 (mon.1) 14 : audit [INF] from='client.? 172.21.15.153:0/2574161419' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]: dispatch 2024-04-23T00:45:51.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:50.246150+0000 mon.smithi121 (mon.0) 476 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]: dispatch 2024-04-23T00:45:51.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:50.254949+0000 mon.smithi121 (mon.0) 477 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c6cb61d-3ca7-4edd-b60d-15287122b534"}]': finished 2024-04-23T00:45:51.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:50.255005+0000 mon.smithi121 (mon.0) 478 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-04-23T00:45:51.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:50.255173+0000 mon.smithi121 (mon.0) 479 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:45:51.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:50.662959+0000 mon.smithi153 (mon.1) 15 : audit [DBG] from='client.? 172.21.15.153:0/3027772601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-23T00:45:52.318 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:51.746097+0000 mgr.smithi121.otezso (mgr.14162) 195 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:52.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:51.746097+0000 mgr.smithi121.otezso (mgr.14162) 195 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:54.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:53.746491+0000 mgr.smithi121.otezso (mgr.14162) 196 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:54.373 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:53.746491+0000 mgr.smithi121.otezso (mgr.14162) 196 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:57.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:55.746904+0000 mgr.smithi121.otezso (mgr.14162) 197 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:57.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:56.467286+0000 mon.smithi121 (mon.0) 480 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T00:45:57.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:45:56.468087+0000 mon.smithi121 (mon.0) 481 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:57.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:55.746904+0000 mgr.smithi121.otezso (mgr.14162) 197 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:57.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:56.467286+0000 mon.smithi121 (mon.0) 480 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T00:45:57.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:45:56.468087+0000 mon.smithi121 (mon.0) 481 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:45:58.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:45:56.468727+0000 mgr.smithi121.otezso (mgr.14162) 198 : cephadm [INF] Deploying daemon osd.7 on smithi153 2024-04-23T00:45:58.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:45:56.468727+0000 mgr.smithi121.otezso (mgr.14162) 198 : cephadm [INF] Deploying daemon osd.7 on smithi153 2024-04-23T00:45:59.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:57.747290+0000 mgr.smithi121.otezso (mgr.14162) 199 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:45:59.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:57.747290+0000 mgr.smithi121.otezso (mgr.14162) 199 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:00.066 INFO:teuthology.orchestra.run.smithi153.stdout:Created osd(s) 7 on host 'smithi153' 2024-04-23T00:46:00.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:45:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:45:59.747645+0000 mgr.smithi121.otezso (mgr.14162) 200 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:00.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:45:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:45:59.747645+0000 mgr.smithi121.otezso (mgr.14162) 200 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:00.722 DEBUG:teuthology.orchestra.run.smithi153:osd.7> sudo journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.7.service 2024-04-23T00:46:00.724 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-23T00:46:00.724 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-23T00:46:00.756 INFO:journalctl@ceph.osd.7.smithi153.stdout:-- Logs begin at Tue 2024-04-23 00:33:27 UTC. -- 2024-04-23T00:46:01.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:00.055198+0000 mon.smithi121 (mon.0) 482 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:46:01.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:00.061205+0000 mon.smithi121 (mon.0) 483 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:01.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:00.067598+0000 mon.smithi121 (mon.0) 484 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:01.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:00.055198+0000 mon.smithi121 (mon.0) 482 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:46:01.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:00.061205+0000 mon.smithi121 (mon.0) 483 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:01.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:00.067598+0000 mon.smithi121 (mon.0) 484 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:02.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:01.748087+0000 mgr.smithi121.otezso (mgr.14162) 201 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:02.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:01.748087+0000 mgr.smithi121.otezso (mgr.14162) 201 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:02.882 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:03.419 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 00:46:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T00:46:03.198+0000 7efd54999080 -1 osd.7 0 log_to_monitors {default=true} 2024-04-23T00:46:03.638 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":46,"num_osds":8,"num_up_osds":7,"osd_up_since":1713833144,"num_in_osds":8,"osd_in_since":1713833150,"num_remapped_pgs":0} 2024-04-23T00:46:03.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:02.739906+0000 mon.smithi121 (mon.0) 485 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:03.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:02.886010+0000 mon.smithi121 (mon.0) 486 : audit [DBG] from='client.? 172.21.15.121:0/1015039099' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-23T00:46:03.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:03.203121+0000 mon.smithi153 (mon.1) 16 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T00:46:03.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:03.203512+0000 mon.smithi121 (mon.0) 487 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T00:46:04.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:02.739906+0000 mon.smithi121 (mon.0) 485 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:04.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:02.886010+0000 mon.smithi121 (mon.0) 486 : audit [DBG] from='client.? 172.21.15.121:0/1015039099' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-23T00:46:04.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:03.203121+0000 mon.smithi153 (mon.1) 16 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T00:46:04.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:03.203512+0000 mon.smithi121 (mon.0) 487 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T00:46:04.639 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-23T00:46:05.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:03.748430+0000 mgr.smithi121.otezso (mgr.14162) 202 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:05.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:04.073208+0000 mon.smithi121 (mon.0) 488 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-23T00:46:05.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:04.073296+0000 mon.smithi121 (mon.0) 489 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-04-23T00:46:05.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:04.073425+0000 mon.smithi121 (mon.0) 490 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:05.059 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:04.073894+0000 mon.smithi153 (mon.1) 17 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:46:05.059 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:04.074151+0000 mon.smithi121 (mon.0) 491 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:03.748430+0000 mgr.smithi121.otezso (mgr.14162) 202 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:04.073208+0000 mon.smithi121 (mon.0) 488 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:04.073296+0000 mon.smithi121 (mon.0) 489 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:04.073425+0000 mon.smithi121 (mon.0) 490 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:04.073894+0000 mon.smithi153 (mon.1) 17 : audit [INF] from='osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:46:05.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:04.074151+0000 mon.smithi121 (mon.0) 491 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T00:46:05.569 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 00:46:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T00:46:05.074+0000 7efd4af0a700 -1 osd.7 0 waiting for initial osdmap 2024-04-23T00:46:05.569 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 00:46:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T00:46:05.081+0000 7efd45682700 -1 osd.7 48 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-04-23T00:46:06.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:05.071474+0000 mon.smithi121 (mon.0) 492 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:46:06.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:05.071532+0000 mon.smithi121 (mon.0) 493 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2024-04-23T00:46:06.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:05.071656+0000 mon.smithi121 (mon.0) 494 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:06.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:05.075865+0000 mon.smithi121 (mon.0) 495 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:06.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:05.748906+0000 mgr.smithi121.otezso (mgr.14162) 203 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:06.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:05.071474+0000 mon.smithi121 (mon.0) 492 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]': finished 2024-04-23T00:46:06.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:05.071532+0000 mon.smithi121 (mon.0) 493 : cluster [DBG] osdmap e48: 8 total, 7 up, 8 in 2024-04-23T00:46:06.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:05.071656+0000 mon.smithi121 (mon.0) 494 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:06.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:05.075865+0000 mon.smithi121 (mon.0) 495 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:06.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:05.748906+0000 mgr.smithi121.otezso (mgr.14162) 203 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-04-23T00:46:06.861 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:07.212 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:04.235223+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:04.235391+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:06.077382+0000 mon.smithi121 (mon.0) 496 : cluster [INF] osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387] boot 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:06.077425+0000 mon.smithi121 (mon.0) 497 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.077551+0000 mon.smithi121 (mon.0) 498 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:46:06.122678+0000 mgr.smithi121.otezso (mgr.14162) 204 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.127360+0000 mon.smithi121 (mon.0) 499 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.130774+0000 mon.smithi121 (mon.0) 500 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.133363+0000 mon.smithi121 (mon.0) 501 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.137797+0000 mon.smithi121 (mon.0) 502 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:06.863641+0000 mon.smithi121 (mon.0) 503 : audit [DBG] from='client.? 172.21.15.121:0/3061295080' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-23T00:46:07.561 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1713833166,"num_in_osds":8,"osd_in_since":1713833150,"num_remapped_pgs":1} 2024-04-23T00:46:07.561 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:04.235223+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:04.235391+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:06.077382+0000 mon.smithi121 (mon.0) 496 : cluster [INF] osd.7 [v2:172.21.15.153:6824/2734942387,v1:172.21.15.153:6825/2734942387] boot 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:06.077425+0000 mon.smithi121 (mon.0) 497 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.077551+0000 mon.smithi121 (mon.0) 498 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:46:06.122678+0000 mgr.smithi121.otezso (mgr.14162) 204 : cephadm [INF] Detected new or changed devices on smithi153 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.127360+0000 mon.smithi121 (mon.0) 499 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.130774+0000 mon.smithi121 (mon.0) 500 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.133363+0000 mon.smithi121 (mon.0) 501 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.137797+0000 mon.smithi121 (mon.0) 502 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:46:07.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:06.863641+0000 mon.smithi121 (mon.0) 503 : audit [DBG] from='client.? 172.21.15.121:0/3061295080' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-23T00:46:07.781 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:08.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:07.077137+0000 mon.smithi121 (mon.0) 504 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-04-23T00:46:08.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:07.749261+0000 mgr.smithi121.otezso (mgr.14162) 205 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:08.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:07.077137+0000 mon.smithi121 (mon.0) 504 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-04-23T00:46:08.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:07.749261+0000 mgr.smithi121.otezso (mgr.14162) 205 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:09.183 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:08.086659+0000 mon.smithi121 (mon.0) 505 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-04-23T00:46:09.539 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:09.539 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":51,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","created":"2024-04-23T00:40:22.936112+0000","modified":"2024-04-23T00:46:08.079236+0000","last_up_change":"2024-04-23T00:46:06.069975+0000","last_in_change":"2024-04-23T00:45:50.246728+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-04-23T00:44:26.316711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"c518eaa9-6bd6-403b-881b-cca9f381b2f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6803","nonce":283218753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6805","nonce":283218753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6809","nonce":283218753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6807","nonce":283218753}]},"public_addr":"172.21.15.121:6803/283218753","cluster_addr":"172.21.15.121:6805/283218753","heartbeat_back_addr":"172.21.15.121:6809/283218753","heartbeat_front_addr":"172.21.15.121:6807/283218753","state":["exists","up"]},{"osd":1,"uuid":"d7f30496-f30d-4737-a044-0ed8ba3f3a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6811","nonce":948167702}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6813","nonce":948167702}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6817","nonce":948167702}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6815","nonce":948167702}]},"public_addr":"172.21.15.121:6811/948167702","cluster_addr":"172.21.15.121:6813/948167702","heartbeat_back_addr":"172.21.15.121:6817/948167702","heartbeat_front_addr":"172.21.15.121:6815/948167702","state":["exists","up"]},{"osd":2,"uuid":"7974c8fd-b1b3-4053-aeb2-fd363e2866c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6819","nonce":4089179417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6821","nonce":4089179417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6825","nonce":4089179417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6823","nonce":4089179417}]},"public_addr":"172.21.15.121:6819/4089179417","cluster_addr":"172.21.15.121:6821/4089179417","heartbeat_back_addr":"172.21.15.121:6825/4089179417","heartbeat_front_addr":"172.21.15.121:6823/4089179417","state":["exists","up"]},{"osd":3,"uuid":"0d2acc81-8c5f-494e-a7d4-20d215a30be5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6827","nonce":3336760544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6829","nonce":3336760544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6833","nonce":3336760544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6831","nonce":3336760544}]},"public_addr":"172.21.15.121:6827/3336760544","cluster_addr":"172.21.15.121:6829/3336760544","heartbeat_back_addr":"172.21.15.121:6833/3336760544","heartbeat_front_addr":"172.21.15.121:6831/3336760544","state":["exists","up"]},{"osd":4,"uuid":"f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6801","nonce":1662652181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6803","nonce":1662652181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6807","nonce":1662652181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6805","nonce":1662652181}]},"public_addr":"172.21.15.153:6801/1662652181","cluster_addr":"172.21.15.153:6803/1662652181","heartbeat_back_addr":"172.21.15.153:6807/1662652181","heartbeat_front_addr":"172.21.15.153:6805/1662652181","state":["exists","up"]},{"osd":5,"uuid":"b2506530-71bf-42e3-9691-1e05ff732924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6809","nonce":3077775392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6811","nonce":3077775392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6815","nonce":3077775392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6813","nonce":3077775392}]},"public_addr":"172.21.15.153:6809/3077775392","cluster_addr":"172.21.15.153:6811/3077775392","heartbeat_back_addr":"172.21.15.153:6815/3077775392","heartbeat_front_addr":"172.21.15.153:6813/3077775392","state":["exists","up"]},{"osd":6,"uuid":"de0de950-d455-48e7-8be6-670f8fead757","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6817","nonce":1407250165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6819","nonce":1407250165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6823","nonce":1407250165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6821","nonce":1407250165}]},"public_addr":"172.21.15.153:6817/1407250165","cluster_addr":"172.21.15.153:6819/1407250165","heartbeat_back_addr":"172.21.15.153:6823/1407250165","heartbeat_front_addr":"172.21.15.153:6821/1407250165","state":["exists","up"]},{"osd":7,"uuid":"4c6cb61d-3ca7-4edd-b60d-15287122b534","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6825","nonce":2734942387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6827","nonce":2734942387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6831","nonce":2734942387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6829","nonce":2734942387}]},"public_addr":"172.21.15.153:6825/2734942387","cluster_addr":"172.21.15.153:6827/2734942387","heartbeat_back_addr":"172.21.15.153:6831/2734942387","heartbeat_front_addr":"172.21.15.153:6829/2734942387","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:43:36.829580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:00.613047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:24.691653+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:46.327956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:03.066081+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:23.292773+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:43.312293+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:46:04.235396+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/1607984048":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/4152529742":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/3464922158":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6801/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:6800/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/1452480097":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/20147015":"2024-04-24T00:41:09.809972+0000","172.21.15.121:0/1894700149":"2024-04-24T00:40:50.776981+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-23T00:46:09.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:08.086659+0000 mon.smithi121 (mon.0) 505 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-04-23T00:46:10.212 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:09.541083+0000 mon.smithi121 (mon.0) 506 : audit [DBG] from='client.? 172.21.15.121:0/116890620' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:10.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:09.749788+0000 mgr.smithi121.otezso (mgr.14162) 206 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:10.229 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2024-04-23T00:44:26.316711+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2024-04-23T00:46:10.230 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd pool get device_health_metrics pg_num 2024-04-23T00:46:10.448 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:10.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:09.541083+0000 mon.smithi121 (mon.0) 506 : audit [DBG] from='client.? 172.21.15.121:0/116890620' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:10.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:09.749788+0000 mgr.smithi121.otezso (mgr.14162) 206 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:12.509 INFO:teuthology.orchestra.run.smithi121.stdout:pg_num: 1 2024-04-23T00:46:13.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:11.750319+0000 mgr.smithi121.otezso (mgr.14162) 207 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:13.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:12.512148+0000 mon.smithi121 (mon.0) 507 : audit [DBG] from='client.? 172.21.15.121:0/3170955537' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-04-23T00:46:13.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:11.750319+0000 mgr.smithi121.otezso (mgr.14162) 207 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:13.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:12.512148+0000 mon.smithi121 (mon.0) 507 : audit [DBG] from='client.? 172.21.15.121:0/3170955537' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-04-23T00:46:13.305 INFO:tasks.cephadm:Setting up client nodes... 2024-04-23T00:46:13.306 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-23T00:46:14.319 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:13.750593+0000 mgr.smithi121.otezso (mgr.14162) 208 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:14.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:13.750593+0000 mgr.smithi121.otezso (mgr.14162) 208 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:15.535 INFO:teuthology.orchestra.run.smithi121.stdout:[client.0] 2024-04-23T00:46:15.536 INFO:teuthology.orchestra.run.smithi121.stdout: key = AQDXBCdmwtTvHxAAwWueluYZJyNead3KU3wzUA== 2024-04-23T00:46:15.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:15.535661+0000 mon.smithi121 (mon.0) 508 : audit [INF] from='client.? 172.21.15.121:0/4159317945' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-23T00:46:15.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:15.538745+0000 mon.smithi121 (mon.0) 509 : audit [INF] from='client.? 172.21.15.121:0/4159317945' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-23T00:46:16.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:15.535661+0000 mon.smithi121 (mon.0) 508 : audit [INF] from='client.? 172.21.15.121:0/4159317945' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-23T00:46:16.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:15.538745+0000 mon.smithi121 (mon.0) 509 : audit [INF] from='client.? 172.21.15.121:0/4159317945' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-23T00:46:16.174 DEBUG:teuthology.orchestra.run.smithi121:> set -ex 2024-04-23T00:46:16.174 DEBUG:teuthology.orchestra.run.smithi121:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-23T00:46:16.174 DEBUG:teuthology.orchestra.run.smithi121:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-23T00:46:16.215 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-23T00:46:16.215 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-23T00:46:16.215 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-23T00:46:16.471 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:16.873 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:15.750898+0000 mgr.smithi121.otezso (mgr.14162) 209 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:17.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:15.750898+0000 mgr.smithi121.otezso (mgr.14162) 209 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:18.351 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:19.053 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:17.751416+0000 mgr.smithi121.otezso (mgr.14162) 210 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:19.054 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:18.349467+0000 mon.smithi121 (mon.0) 510 : audit [DBG] from='client.? 172.21.15.121:0/2501464366' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-23T00:46:19.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:17.751416+0000 mgr.smithi121.otezso (mgr.14162) 210 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:19.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:18.349467+0000 mon.smithi121 (mon.0) 510 : audit [DBG] from='client.? 172.21.15.121:0/2501464366' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-23T00:46:19.071 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":18,"active_gid":14162,"active_name":"smithi121.otezso","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6800","nonce":4004283673},{"type":"v1","addr":"172.21.15.121:6801","nonce":4004283673}]},"active_addr":"172.21.15.121:6801/4004283673","active_change":"2024-04-23T00:41:25.646377+0000","active_mgr_features":4540138297136906239,"available":true,"standbys":[{"gid":14220,"name":"smithi153.olhmgd","mgr_features":4540138297136906239,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"haproxy","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"haproxy","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://smithi121.front.sepia.ceph.com:8443/","prometheus":"http://smithi121.front.sepia.ceph.com:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":4,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":514165074}]},{"addrvec":[{"type":"v2","addr":"172.21.15.121:0","nonce":568295385}]}]}} 2024-04-23T00:46:19.077 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-23T00:46:19.077 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-23T00:46:19.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-23T00:46:19.297 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:20.936 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:19.751736+0000 mgr.smithi121.otezso (mgr.14162) 211 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:21.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:19.751736+0000 mgr.smithi121.otezso (mgr.14162) 211 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:21.324 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:21.324 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":51,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","created":"2024-04-23T00:40:22.936112+0000","modified":"2024-04-23T00:46:08.079236+0000","last_up_change":"2024-04-23T00:46:06.069975+0000","last_in_change":"2024-04-23T00:45:50.246728+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-04-23T00:44:26.316711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"c518eaa9-6bd6-403b-881b-cca9f381b2f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6803","nonce":283218753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6805","nonce":283218753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6809","nonce":283218753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6807","nonce":283218753}]},"public_addr":"172.21.15.121:6803/283218753","cluster_addr":"172.21.15.121:6805/283218753","heartbeat_back_addr":"172.21.15.121:6809/283218753","heartbeat_front_addr":"172.21.15.121:6807/283218753","state":["exists","up"]},{"osd":1,"uuid":"d7f30496-f30d-4737-a044-0ed8ba3f3a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6811","nonce":948167702}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6813","nonce":948167702}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6817","nonce":948167702}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6815","nonce":948167702}]},"public_addr":"172.21.15.121:6811/948167702","cluster_addr":"172.21.15.121:6813/948167702","heartbeat_back_addr":"172.21.15.121:6817/948167702","heartbeat_front_addr":"172.21.15.121:6815/948167702","state":["exists","up"]},{"osd":2,"uuid":"7974c8fd-b1b3-4053-aeb2-fd363e2866c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6819","nonce":4089179417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6821","nonce":4089179417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6825","nonce":4089179417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6823","nonce":4089179417}]},"public_addr":"172.21.15.121:6819/4089179417","cluster_addr":"172.21.15.121:6821/4089179417","heartbeat_back_addr":"172.21.15.121:6825/4089179417","heartbeat_front_addr":"172.21.15.121:6823/4089179417","state":["exists","up"]},{"osd":3,"uuid":"0d2acc81-8c5f-494e-a7d4-20d215a30be5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6827","nonce":3336760544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6829","nonce":3336760544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6833","nonce":3336760544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6831","nonce":3336760544}]},"public_addr":"172.21.15.121:6827/3336760544","cluster_addr":"172.21.15.121:6829/3336760544","heartbeat_back_addr":"172.21.15.121:6833/3336760544","heartbeat_front_addr":"172.21.15.121:6831/3336760544","state":["exists","up"]},{"osd":4,"uuid":"f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6801","nonce":1662652181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6803","nonce":1662652181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6807","nonce":1662652181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6805","nonce":1662652181}]},"public_addr":"172.21.15.153:6801/1662652181","cluster_addr":"172.21.15.153:6803/1662652181","heartbeat_back_addr":"172.21.15.153:6807/1662652181","heartbeat_front_addr":"172.21.15.153:6805/1662652181","state":["exists","up"]},{"osd":5,"uuid":"b2506530-71bf-42e3-9691-1e05ff732924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6809","nonce":3077775392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6811","nonce":3077775392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6815","nonce":3077775392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6813","nonce":3077775392}]},"public_addr":"172.21.15.153:6809/3077775392","cluster_addr":"172.21.15.153:6811/3077775392","heartbeat_back_addr":"172.21.15.153:6815/3077775392","heartbeat_front_addr":"172.21.15.153:6813/3077775392","state":["exists","up"]},{"osd":6,"uuid":"de0de950-d455-48e7-8be6-670f8fead757","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6817","nonce":1407250165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6819","nonce":1407250165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6823","nonce":1407250165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6821","nonce":1407250165}]},"public_addr":"172.21.15.153:6817/1407250165","cluster_addr":"172.21.15.153:6819/1407250165","heartbeat_back_addr":"172.21.15.153:6823/1407250165","heartbeat_front_addr":"172.21.15.153:6821/1407250165","state":["exists","up"]},{"osd":7,"uuid":"4c6cb61d-3ca7-4edd-b60d-15287122b534","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6825","nonce":2734942387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6827","nonce":2734942387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6831","nonce":2734942387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6829","nonce":2734942387}]},"public_addr":"172.21.15.153:6825/2734942387","cluster_addr":"172.21.15.153:6827/2734942387","heartbeat_back_addr":"172.21.15.153:6831/2734942387","heartbeat_front_addr":"172.21.15.153:6829/2734942387","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:43:36.829580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:00.613047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:24.691653+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:46.327956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:03.066081+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:23.292773+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:43.312293+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:46:04.235396+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/1607984048":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/4152529742":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/3464922158":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6801/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:6800/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/1452480097":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/20147015":"2024-04-24T00:41:09.809972+0000","172.21.15.121:0/1894700149":"2024-04-24T00:40:50.776981+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-23T00:46:22.016 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:21.326253+0000 mon.smithi121 (mon.0) 511 : audit [DBG] from='client.? 172.21.15.121:0/2867656433' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:22.035 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-23T00:46:22.035 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-23T00:46:22.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:21.326253+0000 mon.smithi121 (mon.0) 511 : audit [DBG] from='client.? 172.21.15.121:0/2867656433' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:22.254 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:23.066 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:21.752233+0000 mgr.smithi121.otezso (mgr.14162) 212 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:23.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:21.752233+0000 mgr.smithi121.otezso (mgr.14162) 212 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:24.112 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:46:24.112 INFO:teuthology.orchestra.run.smithi121.stdout:{"epoch":51,"fsid":"004558ac-010a-11ef-bc93-c7b262605968","created":"2024-04-23T00:40:22.936112+0000","modified":"2024-04-23T00:46:08.079236+0000","last_up_change":"2024-04-23T00:46:06.069975+0000","last_in_change":"2024-04-23T00:45:50.246728+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-04-23T00:44:26.316711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"c518eaa9-6bd6-403b-881b-cca9f381b2f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6802","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6803","nonce":283218753}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6804","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6805","nonce":283218753}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6808","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6809","nonce":283218753}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6806","nonce":283218753},{"type":"v1","addr":"172.21.15.121:6807","nonce":283218753}]},"public_addr":"172.21.15.121:6803/283218753","cluster_addr":"172.21.15.121:6805/283218753","heartbeat_back_addr":"172.21.15.121:6809/283218753","heartbeat_front_addr":"172.21.15.121:6807/283218753","state":["exists","up"]},{"osd":1,"uuid":"d7f30496-f30d-4737-a044-0ed8ba3f3a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6810","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6811","nonce":948167702}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6812","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6813","nonce":948167702}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6816","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6817","nonce":948167702}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6814","nonce":948167702},{"type":"v1","addr":"172.21.15.121:6815","nonce":948167702}]},"public_addr":"172.21.15.121:6811/948167702","cluster_addr":"172.21.15.121:6813/948167702","heartbeat_back_addr":"172.21.15.121:6817/948167702","heartbeat_front_addr":"172.21.15.121:6815/948167702","state":["exists","up"]},{"osd":2,"uuid":"7974c8fd-b1b3-4053-aeb2-fd363e2866c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6818","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6819","nonce":4089179417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6820","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6821","nonce":4089179417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6824","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6825","nonce":4089179417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6822","nonce":4089179417},{"type":"v1","addr":"172.21.15.121:6823","nonce":4089179417}]},"public_addr":"172.21.15.121:6819/4089179417","cluster_addr":"172.21.15.121:6821/4089179417","heartbeat_back_addr":"172.21.15.121:6825/4089179417","heartbeat_front_addr":"172.21.15.121:6823/4089179417","state":["exists","up"]},{"osd":3,"uuid":"0d2acc81-8c5f-494e-a7d4-20d215a30be5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6826","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6827","nonce":3336760544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6828","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6829","nonce":3336760544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6832","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6833","nonce":3336760544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.121:6830","nonce":3336760544},{"type":"v1","addr":"172.21.15.121:6831","nonce":3336760544}]},"public_addr":"172.21.15.121:6827/3336760544","cluster_addr":"172.21.15.121:6829/3336760544","heartbeat_back_addr":"172.21.15.121:6833/3336760544","heartbeat_front_addr":"172.21.15.121:6831/3336760544","state":["exists","up"]},{"osd":4,"uuid":"f1ac5d29-5ef0-4204-b49d-9b8d49a5c81e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6800","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6801","nonce":1662652181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6803","nonce":1662652181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6807","nonce":1662652181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":1662652181},{"type":"v1","addr":"172.21.15.153:6805","nonce":1662652181}]},"public_addr":"172.21.15.153:6801/1662652181","cluster_addr":"172.21.15.153:6803/1662652181","heartbeat_back_addr":"172.21.15.153:6807/1662652181","heartbeat_front_addr":"172.21.15.153:6805/1662652181","state":["exists","up"]},{"osd":5,"uuid":"b2506530-71bf-42e3-9691-1e05ff732924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6809","nonce":3077775392}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6810","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6811","nonce":3077775392}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6814","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6815","nonce":3077775392}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6812","nonce":3077775392},{"type":"v1","addr":"172.21.15.153:6813","nonce":3077775392}]},"public_addr":"172.21.15.153:6809/3077775392","cluster_addr":"172.21.15.153:6811/3077775392","heartbeat_back_addr":"172.21.15.153:6815/3077775392","heartbeat_front_addr":"172.21.15.153:6813/3077775392","state":["exists","up"]},{"osd":6,"uuid":"de0de950-d455-48e7-8be6-670f8fead757","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":43,"up_thru":44,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6816","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6817","nonce":1407250165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6818","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6819","nonce":1407250165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6822","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6823","nonce":1407250165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6820","nonce":1407250165},{"type":"v1","addr":"172.21.15.153:6821","nonce":1407250165}]},"public_addr":"172.21.15.153:6817/1407250165","cluster_addr":"172.21.15.153:6819/1407250165","heartbeat_back_addr":"172.21.15.153:6823/1407250165","heartbeat_front_addr":"172.21.15.153:6821/1407250165","state":["exists","up"]},{"osd":7,"uuid":"4c6cb61d-3ca7-4edd-b60d-15287122b534","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6824","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6825","nonce":2734942387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6826","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6827","nonce":2734942387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6830","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6831","nonce":2734942387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6828","nonce":2734942387},{"type":"v1","addr":"172.21.15.153:6829","nonce":2734942387}]},"public_addr":"172.21.15.153:6825/2734942387","cluster_addr":"172.21.15.153:6827/2734942387","heartbeat_back_addr":"172.21.15.153:6831/2734942387","heartbeat_front_addr":"172.21.15.153:6829/2734942387","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:43:36.829580+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:00.613047+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:24.691653+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:44:46.327956+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:03.066081+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:23.292773+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:45:43.312293+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-04-23T00:46:04.235396+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.121:0/1607984048":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/4152529742":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:0/3464922158":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6801/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/1739204640":"2024-04-24T00:41:09.809972+0000","172.21.15.121:6800/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:6800/1320354118":"2024-04-24T00:41:25.646096+0000","172.21.15.121:6801/2073913881":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/1452480097":"2024-04-24T00:40:50.776981+0000","172.21.15.121:0/20147015":"2024-04-24T00:41:09.809972+0000","172.21.15.121:0/1894700149":"2024-04-24T00:40:50.776981+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-23T00:46:24.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:23.752524+0000 mgr.smithi121.otezso (mgr.14162) 213 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:24.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:23.752524+0000 mgr.smithi121.otezso (mgr.14162) 213 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:24.796 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-23T00:46:24.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-23T00:46:24.797 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-23T00:46:24.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-23T00:46:24.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-23T00:46:24.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-23T00:46:24.799 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-23T00:46:24.799 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-23T00:46:25.083 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:24.115006+0000 mon.smithi121 (mon.0) 512 : audit [DBG] from='client.? 172.21.15.121:0/311177592' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:25.173 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:25.224 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:25.239 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:25.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:24.115006+0000 mon.smithi121 (mon.0) 512 : audit [DBG] from='client.? 172.21.15.121:0/311177592' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-23T00:46:26.178 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:26.179 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:26.189 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:26.190 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:26.191 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:26.353 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:25.752795+0000 mgr.smithi121.otezso (mgr.14162) 214 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:26.353 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:25.855221+0000 mon.smithi121 (mon.0) 513 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:46:26.353 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:25.855404+0000 mon.smithi121 (mon.0) 514 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:46:26.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:25.752795+0000 mgr.smithi121.otezso (mgr.14162) 214 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:26.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:25.855221+0000 mon.smithi121 (mon.0) 513 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:46:26.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:25.855404+0000 mon.smithi121 (mon.0) 514 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:46:28.989 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:27.753279+0000 mgr.smithi121.otezso (mgr.14162) 215 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:29.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:27.753279+0000 mgr.smithi121.otezso (mgr.14162) 215 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:31.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:29.753562+0000 mgr.smithi121.otezso (mgr.14162) 216 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:31.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:29.753562+0000 mgr.smithi121.otezso (mgr.14162) 216 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:32.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:31.753999+0000 mgr.smithi121.otezso (mgr.14162) 217 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:32.255 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:31.753999+0000 mgr.smithi121.otezso (mgr.14162) 217 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:34.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:33.754416+0000 mgr.smithi121.otezso (mgr.14162) 218 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:34.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:33.754416+0000 mgr.smithi121.otezso (mgr.14162) 218 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:36.077 INFO:teuthology.orchestra.run.smithi121.stdout:60129542177 2024-04-23T00:46:36.078 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-23T00:46:36.925 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:35.754741+0000 mgr.smithi121.otezso (mgr.14162) 219 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:37.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:35.754741+0000 mgr.smithi121.otezso (mgr.14162) 219 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:38.249 INFO:teuthology.orchestra.run.smithi121.stdout:133143986197 2024-04-23T00:46:38.249 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-23T00:46:39.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:37.755253+0000 mgr.smithi121.otezso (mgr.14162) 220 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:39.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:37.755253+0000 mgr.smithi121.otezso (mgr.14162) 220 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:39.278 INFO:teuthology.orchestra.run.smithi121.stdout:111669149721 2024-04-23T00:46:39.278 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-23T00:46:40.023 INFO:teuthology.orchestra.run.smithi121.stdout:158913789970 2024-04-23T00:46:40.023 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-23T00:46:41.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:39.755723+0000 mgr.smithi121.otezso (mgr.14162) 221 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:41.080 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:39.755723+0000 mgr.smithi121.otezso (mgr.14162) 221 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:41.096 INFO:teuthology.orchestra.run.smithi121.stdout:38654705703 2024-04-23T00:46:41.096 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-23T00:46:41.403 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:41.403 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:41.403 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:41.404 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:41.432 INFO:teuthology.orchestra.run.smithi121.stdout:210453397513 2024-04-23T00:46:41.432 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-23T00:46:41.932 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:41.756303+0000 mgr.smithi121.otezso (mgr.14162) 222 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:42.255 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:41.756303+0000 mgr.smithi121.otezso (mgr.14162) 222 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:43.225 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:43.814 INFO:teuthology.orchestra.run.smithi121.stdout:81604378654 2024-04-23T00:46:43.814 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-23T00:46:44.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:43.756762+0000 mgr.smithi121.otezso (mgr.14162) 223 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:44.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:43.756762+0000 mgr.smithi121.otezso (mgr.14162) 223 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:44.694 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:46.921 INFO:teuthology.orchestra.run.smithi121.stdout:133143986199 2024-04-23T00:46:47.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:45.757160+0000 mgr.smithi121.otezso (mgr.14162) 224 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:47.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:46.068103+0000 mon.smithi121 (mon.0) 515 : audit [DBG] from='client.? 172.21.15.121:0/1614312212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-23T00:46:47.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:45.757160+0000 mgr.smithi121.otezso (mgr.14162) 224 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:47.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:46.068103+0000 mon.smithi121 (mon.0) 515 : audit [DBG] from='client.? 172.21.15.121:0/1614312212' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-23T00:46:47.723 INFO:teuthology.orchestra.run.smithi121.stdout:184683593742 2024-04-23T00:46:47.724 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-23T00:46:49.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:47.757481+0000 mgr.smithi121.otezso (mgr.14162) 225 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:49.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:47.757481+0000 mgr.smithi121.otezso (mgr.14162) 225 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:49.358 INFO:teuthology.orchestra.run.smithi121.stdout:60129542180 2024-04-23T00:46:50.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:49.360713+0000 mon.smithi121 (mon.0) 516 : audit [DBG] from='client.? 172.21.15.121:0/594112240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-23T00:46:50.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:49.360713+0000 mon.smithi121 (mon.0) 516 : audit [DBG] from='client.? 172.21.15.121:0/594112240' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-23T00:46:51.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:49.758008+0000 mgr.smithi121.otezso (mgr.14162) 226 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:51.114 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:49.758008+0000 mgr.smithi121.otezso (mgr.14162) 226 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:51.612 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542177 got 60129542180 for osd.1 2024-04-23T00:46:51.612 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:52.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:51.758415+0000 mgr.smithi121.otezso (mgr.14162) 227 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:52.236 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:52.255 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:51.758415+0000 mgr.smithi121.otezso (mgr.14162) 227 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:52.265 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986197 got 133143986199 for osd.4 2024-04-23T00:46:52.265 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:52.505 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:46:54.059 INFO:teuthology.orchestra.run.smithi121.stdout:210453397516 2024-04-23T00:46:54.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:53.682861+0000 mon.smithi121 (mon.0) 517 : audit [DBG] from='client.? 172.21.15.121:0/2312005534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-23T00:46:54.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:53.758685+0000 mgr.smithi121.otezso (mgr.14162) 228 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:54.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:53.682861+0000 mon.smithi121 (mon.0) 517 : audit [DBG] from='client.? 172.21.15.121:0/2312005534' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-23T00:46:54.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:53.758685+0000 mgr.smithi121.otezso (mgr.14162) 228 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:54.622 INFO:teuthology.orchestra.run.smithi121.stdout:38654705706 2024-04-23T00:46:54.824 INFO:teuthology.orchestra.run.smithi121.stdout:111669149725 2024-04-23T00:46:55.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:54.625392+0000 mon.smithi121 (mon.0) 518 : audit [DBG] from='client.? 172.21.15.121:0/1735511918' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-23T00:46:55.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:54.826633+0000 mon.smithi121 (mon.0) 519 : audit [DBG] from='client.? 172.21.15.121:0/1579997091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-23T00:46:55.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:54.625392+0000 mon.smithi121 (mon.0) 518 : audit [DBG] from='client.? 172.21.15.121:0/1735511918' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-23T00:46:55.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:54.826633+0000 mon.smithi121 (mon.0) 519 : audit [DBG] from='client.? 172.21.15.121:0/1579997091' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-23T00:46:55.668 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149721 got 111669149725 for osd.3 2024-04-23T00:46:55.668 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:56.111 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705703 got 38654705706 for osd.0 2024-04-23T00:46:56.111 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:56.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:55.759080+0000 mgr.smithi121.otezso (mgr.14162) 229 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:56.569 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:55.759080+0000 mgr.smithi121.otezso (mgr.14162) 229 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:57.192 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397513 got 210453397516 for osd.7 2024-04-23T00:46:57.193 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:57.897 INFO:teuthology.orchestra.run.smithi121.stdout:158913789974 2024-04-23T00:46:58.497 INFO:teuthology.orchestra.run.smithi121.stdout:184683593745 2024-04-23T00:46:59.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:57.759397+0000 mgr.smithi121.otezso (mgr.14162) 230 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:59.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:57.899579+0000 mon.smithi121 (mon.0) 520 : audit [DBG] from='client.? 172.21.15.121:0/72658926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-23T00:46:59.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:46:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:46:58.499617+0000 mon.smithi121 (mon.0) 521 : audit [DBG] from='client.? 172.21.15.121:0/390072023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-23T00:46:59.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:57.759397+0000 mgr.smithi121.otezso (mgr.14162) 230 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:46:59.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:57.899579+0000 mon.smithi121 (mon.0) 520 : audit [DBG] from='client.? 172.21.15.121:0/72658926' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-23T00:46:59.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:46:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:46:58.499617+0000 mon.smithi121 (mon.0) 521 : audit [DBG] from='client.? 172.21.15.121:0/390072023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-23T00:46:59.245 INFO:tasks.cephadm.ceph_manager.ceph:need seq 184683593742 got 184683593745 for osd.6 2024-04-23T00:46:59.245 DEBUG:teuthology.parallel:result is None 2024-04-23T00:46:59.556 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789970 got 158913789974 for osd.5 2024-04-23T00:46:59.557 DEBUG:teuthology.parallel:result is None 2024-04-23T00:47:00.577 INFO:teuthology.orchestra.run.smithi121.stdout:81604378657 2024-04-23T00:47:00.874 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:46:59.759728+0000 mgr.smithi121.otezso (mgr.14162) 231 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:00.875 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:00.579135+0000 mon.smithi121 (mon.0) 522 : audit [DBG] from='client.? 172.21.15.121:0/2130086234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-23T00:47:01.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:46:59.759728+0000 mgr.smithi121.otezso (mgr.14162) 231 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:01.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:00.579135+0000 mon.smithi121 (mon.0) 522 : audit [DBG] from='client.? 172.21.15.121:0/2130086234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-23T00:47:01.240 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378654 got 81604378657 for osd.2 2024-04-23T00:47:01.240 DEBUG:teuthology.parallel:result is None 2024-04-23T00:47:01.241 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-23T00:47:01.241 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-23T00:47:01.463 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:47:02.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:01.760035+0000 mgr.smithi121.otezso (mgr.14162) 232 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:02.256 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:01.760035+0000 mgr.smithi121.otezso (mgr.14162) 232 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:03.298 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:03.300 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2024-04-23T00:47:03.970 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":208,"stamp":"2024-04-23T00:47:01.759912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43376,"kb_used_data":3888,"kb_used_omap":0,"kb_used_meta":39424,"kb_avail":749917840,"statfs":{"total":767960285184,"available":767915868160,"internally_reserved":0,"allocated":3981312,"data_stored":1588180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40370176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":"10","reported_epoch":"51","state":"active+clean","last_fresh":"2024-04-23T00:46:08.384881+0000","last_change":"2024-04-23T00:46:08.384881+0000","last_active":"2024-04-23T00:46:08.384881+0000","last_peered":"2024-04-23T00:46:08.384881+0000","last_clean":"2024-04-23T00:46:08.384881+0000","last_became_active":"2024-04-23T00:46:08.384569+0000","last_became_peered":"2024-04-23T00:46:08.384569+0000","last_unstale":"2024-04-23T00:46:08.384881+0000","last_undegraded":"2024-04-23T00:46:08.384881+0000","last_fullsized":"2024-04-23T00:46:08.384881+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-23T00:44:27.061264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-23T00:44:27.061264+0000","last_clean_scrub_stamp":"2024-04-23T00:44:27.061264+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397518,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5232,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739920,"statfs":{"total":95995035648,"available":95989678080,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46400000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593746,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5296,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739856,"statfs":{"total":95995035648,"available":95989612544,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.55100000000000005}]},{"osd":1,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.372},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.34300000000000003}]},{"osd":2,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46600000000000003}]},{"osd":3,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.57699999999999996}]},{"osd":4,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.5},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.48999999999999999}]},{"osd":5,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542183,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5548,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739604,"statfs":{"total":95995035648,"available":95989354496,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:03 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34799999999999998,"15min":0.34399999999999997},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.52100000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36199999999999999},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56299999999999994,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.29599999999999999}]},{"osd":2,"last update":"Tue Apr 23 00:46:31 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.378},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53500000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.31}]},{"osd":3,"last update":"Tue Apr 23 00:46:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41499999999999998,"15min":0.41799999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.42599999999999999,"15min":0.42799999999999999},"min":{"1min":0.309,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.55700000000000005,"5min":0.627,"15min":0.627},"last":0.34799999999999998}]},{"osd":4,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51300000000000001}]},{"osd":5,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.69699999999999995}]},{"osd":6,"last update":"Tue Apr 23 00:46:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705708,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.308,"15min":0.30299999999999999},"min":{"1min":0.20599999999999999,"5min":0.192,"15min":0.192},"max":{"1min":0.49099999999999999,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.312,"15min":0.30199999999999999},"min":{"1min":0.23499999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.35899999999999999}]},{"osd":2,"last update":"Tue Apr 23 00:46:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.33400000000000002,"15min":0.33200000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"last":0.378},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.34000000000000002,"15min":0.33800000000000002},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.33100000000000002}]},{"osd":3,"last update":"Tue Apr 23 00:46:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.36099999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.56100000000000005,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.313},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.34200000000000003}]},{"osd":4,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.49199999999999999}]},{"osd":5,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Tue Apr 23 00:46:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.47699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":2,"up_from":19,"seq":81604378657,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5484,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739668,"statfs":{"total":95995035648,"available":95989420032,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.35899999999999999,"15min":0.35599999999999998},"min":{"1min":0.26700000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.34699999999999998}]},{"osd":1,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.35999999999999999,"15min":0.35699999999999998},"min":{"1min":0.28899999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.47699999999999998,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.378}]},{"osd":3,"last update":"Tue Apr 23 00:46:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.48199999999999998,"15min":0.498},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53400000000000003,"5min":1.454,"15min":1.454},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.42399999999999999,"15min":0.42999999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.52000000000000002,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.39000000000000001}]},{"osd":4,"last update":"Tue Apr 23 00:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.41799999999999998}]},{"osd":5,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Tue Apr 23 00:46:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149726,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5424,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739728,"statfs":{"total":95995035648,"available":95989481472,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.34599999999999997,"15min":0.34799999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.36399999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.47499999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.40899999999999997}]},{"osd":1,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.47299999999999998,"5min":0.51300000000000001,"15min":0.51300000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.377,"15min":0.379},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.53900000000000003,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42399999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52100000000000002,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.377,"5min":0.40100000000000002,"15min":0.40500000000000003},"min":{"1min":0.26300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.378}]},{"osd":4,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.499},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53100000000000003}]},{"osd":5,"last update":"Tue Apr 23 00:46:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.56299999999999994}]},{"osd":6,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986202,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5420,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739732,"statfs":{"total":95995035648,"available":95989485568,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.39800000000000002}]},{"osd":1,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.49399999999999999}]},{"osd":2,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.35799999999999998}]},{"osd":3,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.57899999999999996}]},{"osd":5,"last update":"Tue Apr 23 00:46:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.54500000000000004}]},{"osd":6,"last update":"Tue Apr 23 00:46:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.432}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":5,"up_from":37,"seq":158913789975,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5356,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739796,"statfs":{"total":95995035648,"available":95989551104,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":10.898,"5min":10.898,"15min":10.898},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.8579999999999997,"5min":7.8579999999999997,"15min":7.8579999999999997},"last":0.47499999999999998}]},{"osd":1,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.952,"5min":10.952,"15min":10.952},"last":0.503}]},{"osd":2,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":6.0289999999999999,"5min":6.0289999999999999,"15min":6.0289999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":6.3280000000000003,"5min":6.3280000000000003,"15min":6.3280000000000003},"last":0.439}]},{"osd":3,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.1950000000000003,"5min":6.1950000000000003,"15min":6.1950000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":7.8170000000000002,"5min":7.8170000000000002,"15min":7.8170000000000002},"last":0.55900000000000005}]},{"osd":4,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.57299999999999995}]},{"osd":6,"last update":"Tue Apr 23 00:46:48 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56599999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-23T00:47:03.971 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-23T00:47:04.194 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:47:04.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:03.298541+0000 mgr.smithi121.otezso (mgr.14162) 233 : audit [DBG] from='client.14464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:04.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:03.760467+0000 mgr.smithi121.otezso (mgr.14162) 234 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:04.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:03.298541+0000 mgr.smithi121.otezso (mgr.14162) 233 : audit [DBG] from='client.14464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:04.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:03.760467+0000 mgr.smithi121.otezso (mgr.14162) 234 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:05.989 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:05.991 INFO:teuthology.orchestra.run.smithi121.stderr:dumped all 2024-04-23T00:47:06.673 INFO:teuthology.orchestra.run.smithi121.stdout:{"pg_ready":true,"pg_map":{"version":210,"stamp":"2024-04-23T00:47:05.760623+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43376,"kb_used_data":3888,"kb_used_omap":0,"kb_used_meta":39424,"kb_avail":749917840,"statfs":{"total":767960285184,"available":767915868160,"internally_reserved":0,"allocated":3981312,"data_stored":1588180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40370176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":"10","reported_epoch":"51","state":"active+clean","last_fresh":"2024-04-23T00:46:08.384881+0000","last_change":"2024-04-23T00:46:08.384881+0000","last_active":"2024-04-23T00:46:08.384881+0000","last_peered":"2024-04-23T00:46:08.384881+0000","last_clean":"2024-04-23T00:46:08.384881+0000","last_became_active":"2024-04-23T00:46:08.384569+0000","last_became_peered":"2024-04-23T00:46:08.384569+0000","last_unstale":"2024-04-23T00:46:08.384881+0000","last_undegraded":"2024-04-23T00:46:08.384881+0000","last_fullsized":"2024-04-23T00:46:08.384881+0000","mapping_epoch":50,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":51,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-23T00:44:27.061264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-23T00:44:27.061264+0000","last_clean_scrub_stamp":"2024-04-23T00:44:27.061264+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":49,"seq":210453397518,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5232,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739920,"statfs":{"total":95995035648,"available":95989678080,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46400000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]}]},{"osd":6,"up_from":43,"seq":184683593747,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5296,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739856,"statfs":{"total":95995035648,"available":95989612544,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.434},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.52700000000000002}]},{"osd":1,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47799999999999998}]},{"osd":2,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.49299999999999999}]},{"osd":3,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.505},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.40100000000000002}]},{"osd":4,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.35599999999999998}]},{"osd":5,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.443},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.41599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53900000000000003}]}]},{"osd":1,"up_from":14,"seq":60129542184,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5548,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739604,"statfs":{"total":95995035648,"available":95989354496,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:03 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.34799999999999998,"15min":0.34399999999999997},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.52100000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36199999999999999},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56299999999999994,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.32200000000000001}]},{"osd":2,"last update":"Tue Apr 23 00:46:31 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.378},"min":{"1min":0.314,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53500000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.373,"15min":0.36899999999999999},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.308}]},{"osd":3,"last update":"Tue Apr 23 00:46:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41499999999999998,"15min":0.41799999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.42599999999999999,"15min":0.42799999999999999},"min":{"1min":0.309,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.55700000000000005,"5min":0.627,"15min":0.627},"last":0.33100000000000002}]},{"osd":4,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.54200000000000004}]},{"osd":5,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.52400000000000002}]},{"osd":6,"last update":"Tue Apr 23 00:46:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59599999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]}]},{"osd":0,"up_from":9,"seq":38654705709,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.308,"15min":0.30299999999999999},"min":{"1min":0.20599999999999999,"5min":0.192,"15min":0.192},"max":{"1min":0.49099999999999999,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.312,"15min":0.30199999999999999},"min":{"1min":0.23499999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.32200000000000001}]},{"osd":2,"last update":"Tue Apr 23 00:46:32 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.33400000000000002,"15min":0.33200000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.34000000000000002,"15min":0.33800000000000002},"min":{"1min":0.245,"5min":0.219,"15min":0.219},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.36899999999999999}]},{"osd":3,"last update":"Tue Apr 23 00:46:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.36099999999999999,"15min":0.36399999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.56100000000000005,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.31}]},{"osd":4,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.42799999999999999}]},{"osd":5,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.378}]},{"osd":6,"last update":"Tue Apr 23 00:46:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.41699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378658,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5484,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739668,"statfs":{"total":95995035648,"available":95989420032,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.35899999999999999,"15min":0.35599999999999998},"min":{"1min":0.26700000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.378},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.249}]},{"osd":1,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.35999999999999999,"15min":0.35699999999999998},"min":{"1min":0.28899999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.47699999999999998,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Tue Apr 23 00:46:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.48199999999999998,"15min":0.498},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53400000000000003,"5min":1.454,"15min":1.454},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.42399999999999999,"15min":0.42999999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.52000000000000002,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.29499999999999998}]},{"osd":4,"last update":"Tue Apr 23 00:46:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.499},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60899999999999999}]},{"osd":5,"last update":"Tue Apr 23 00:46:30 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59799999999999998}]},{"osd":6,"last update":"Tue Apr 23 00:46:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.59099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149727,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5424,"kb_used_data":488,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739728,"statfs":{"total":95995035648,"available":95989481472,"internally_reserved":0,"allocated":499712,"data_stored":198680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.34599999999999997,"15min":0.34799999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.36399999999999999,"15min":0.36699999999999999},"min":{"1min":0.24199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.47499999999999998,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.46300000000000002}]},{"osd":1,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36599999999999999,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.47299999999999998,"5min":0.51300000000000001,"15min":0.51300000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.377,"15min":0.379},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.53900000000000003,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40300000000000002}]},{"osd":2,"last update":"Tue Apr 23 00:46:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42399999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52100000000000002,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.439},{"interface":"front","average":{"1min":0.377,"5min":0.40100000000000002,"15min":0.40500000000000003},"min":{"1min":0.26300000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.45000000000000001}]},{"osd":4,"last update":"Tue Apr 23 00:46:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.56299999999999994}]},{"osd":5,"last update":"Tue Apr 23 00:46:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64600000000000002}]},{"osd":6,"last update":"Tue Apr 23 00:46:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.752},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.61599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71099999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5420,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739732,"statfs":{"total":95995035648,"available":95989485568,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.49199999999999999}]},{"osd":1,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.54000000000000004}]},{"osd":2,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.5}]},{"osd":3,"last update":"Tue Apr 23 00:46:05 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.55300000000000005}]},{"osd":5,"last update":"Tue Apr 23 00:46:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.46500000000000002}]},{"osd":6,"last update":"Tue Apr 23 00:46:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]}]},{"osd":5,"up_from":37,"seq":158913789975,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5356,"kb_used_data":484,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739796,"statfs":{"total":95995035648,"available":95989551104,"internally_reserved":0,"allocated":495616,"data_stored":198365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":10.898,"5min":10.898,"15min":10.898},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.8579999999999997,"5min":7.8579999999999997,"15min":7.8579999999999997},"last":0.47499999999999998}]},{"osd":1,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":10.952,"5min":10.952,"15min":10.952},"last":0.503}]},{"osd":2,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":6.0289999999999999,"5min":6.0289999999999999,"15min":6.0289999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":6.3280000000000003,"5min":6.3280000000000003,"15min":6.3280000000000003},"last":0.439}]},{"osd":3,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.1950000000000003,"5min":6.1950000000000003,"15min":6.1950000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":7.8170000000000002,"5min":7.8170000000000002,"15min":7.8170000000000002},"last":0.55900000000000005}]},{"osd":4,"last update":"Tue Apr 23 00:46:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.57299999999999995}]},{"osd":6,"last update":"Tue Apr 23 00:46:48 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56599999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-23T00:47:06.674 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-23T00:47:06.674 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-23T00:47:06.674 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-23T00:47:06.674 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-23T00:47:06.894 INFO:teuthology.orchestra.run.smithi121.stderr:Inferring config /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/mon.smithi121/config 2024-04-23T00:47:07.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:05.760760+0000 mgr.smithi121.otezso (mgr.14162) 235 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:07.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:06.140454+0000 mon.smithi121 (mon.0) 523 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:07.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:06.394209+0000 mon.smithi121 (mon.0) 524 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:07.085 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:05.760760+0000 mgr.smithi121.otezso (mgr.14162) 235 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:07.085 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:06.140454+0000 mon.smithi121 (mon.0) 523 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:07.085 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:06.394209+0000 mon.smithi121 (mon.0) 524 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:07.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:05.989329+0000 mgr.smithi121.otezso (mgr.14162) 236 : audit [DBG] from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:08.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:05.989329+0000 mgr.smithi121.otezso (mgr.14162) 236 : audit [DBG] from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:08.674 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:09.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:07.761290+0000 mgr.smithi121.otezso (mgr.14162) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:09.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:08.676037+0000 mon.smithi121 (mon.0) 525 : audit [DBG] from='client.? 172.21.15.121:0/891529056' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-23T00:47:09.096 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:07.761290+0000 mgr.smithi121.otezso (mgr.14162) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:09.096 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:08.676037+0000 mon.smithi121 (mon.0) 525 : audit [DBG] from='client.? 172.21.15.121:0/891529056' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-23T00:47:09.385 INFO:teuthology.orchestra.run.smithi121.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-23T00:47:09.385 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-23T00:47:09.386 INFO:tasks.cephadm:Setup complete, yielding 2024-04-23T00:47:09.386 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-23T00:47:09.400 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:09.403 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-23T00:47:10.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:09.761569+0000 mgr.smithi121.otezso (mgr.14162) 238 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:10.124 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:09.761569+0000 mgr.smithi121.otezso (mgr.14162) 238 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:11.638 INFO:teuthology.orchestra.run.smithi121.stdout:Backend: cephadm 2024-04-23T00:47:11.638 INFO:teuthology.orchestra.run.smithi121.stdout:Available: Yes 2024-04-23T00:47:11.638 INFO:teuthology.orchestra.run.smithi121.stdout:Paused: No 2024-04-23T00:47:12.404 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-23T00:47:13.065 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:11.639921+0000 mgr.smithi121.otezso (mgr.14162) 239 : audit [DBG] from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:13.065 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:11.761928+0000 mgr.smithi121.otezso (mgr.14162) 240 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:13.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:11.639921+0000 mgr.smithi121.otezso (mgr.14162) 239 : audit [DBG] from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:13.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:11.761928+0000 mgr.smithi121.otezso (mgr.14162) 240 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 running (3m) 90s ago 5m *:9093 *:9094 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (5m) 90s ago 5m - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (4m) 71s ago 4m - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 running (3m) 90s ago 5m *:3000 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 running (6m) 90s ago 6m *:9283 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:47:14.373 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 running (4m) 71s ago 4m *:8443 *:9283 16.2.0 24ecd6d5f14c 29976d00060b 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (6m) 90s ago 6m - 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (4m) 71s ago 4m - 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 running (4m) 90s ago 4m *:9100 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 running (4m) 71s ago 4m *:9100 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (3m) 90s ago 3m - 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (3m) 90s ago 3m - 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (2m) 90s ago 2m - 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (2m) 90s ago 2m - 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (2m) 71s ago 2m - 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (115s) 71s ago 115s - 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (95s) 71s ago 95s - 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (74s) 71s ago 74s - 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:47:14.374 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 running (3m) 90s ago 4m *:9095 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:47:14.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:13.762201+0000 mgr.smithi121.otezso (mgr.14162) 241 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:14.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:13.762201+0000 mgr.smithi121.otezso (mgr.14162) 241 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:15.068 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-23T00:47:15.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:14.368629+0000 mgr.smithi121.otezso (mgr.14162) 242 : audit [DBG] from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:15.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:14.368629+0000 mgr.smithi121.otezso (mgr.14162) 242 : audit [DBG] from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:16.374 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:15.762530+0000 mgr.smithi121.otezso (mgr.14162) 243 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:16.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:15.762530+0000 mgr.smithi121.otezso (mgr.14162) 243 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:17.197 INFO:teuthology.orchestra.run.smithi121.stdout:NAME RUNNING REFRESHED AGE PLACEMENT IMAGE ID 2024-04-23T00:47:17.197 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager 1/1 93s ago 6m count:1 0881eb8f169f 2024-04-23T00:47:17.197 INFO:teuthology.orchestra.run.smithi121.stdout:crash 2/2 93s ago 6m * 24ecd6d5f14c 2024-04-23T00:47:17.197 INFO:teuthology.orchestra.run.smithi121.stdout:grafana 1/1 93s ago 6m count:1 557c83e11646 2024-04-23T00:47:17.197 INFO:teuthology.orchestra.run.smithi121.stdout:mgr 2/2 93s ago 6m count:2 24ecd6d5f14c 2024-04-23T00:47:17.198 INFO:teuthology.orchestra.run.smithi121.stdout:mon 2/2 93s ago 5m smithi121:172.21.15.121=smithi121;smithi153:172.21.15.153=smithi153;count:2 24ecd6d5f14c 2024-04-23T00:47:17.198 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter 2/2 93s ago 6m * e5a616e4b9cf 2024-04-23T00:47:17.198 INFO:teuthology.orchestra.run.smithi121.stdout:osd.None 7/0 93s ago - 24ecd6d5f14c 2024-04-23T00:47:17.198 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus 1/1 93s ago 6m count:1 de242295e225 2024-04-23T00:47:17.928 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-23T00:47:19.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:17.195286+0000 mgr.smithi121.otezso (mgr.14162) 244 : audit [DBG] from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:19.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:17.763084+0000 mgr.smithi121.otezso (mgr.14162) 245 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:19.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:17.195286+0000 mgr.smithi121.otezso (mgr.14162) 244 : audit [DBG] from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:19.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:17.763084+0000 mgr.smithi121.otezso (mgr.14162) 245 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:19.967 INFO:teuthology.orchestra.run.smithi121.stdout:HOST ADDR LABELS STATUS 2024-04-23T00:47:19.967 INFO:teuthology.orchestra.run.smithi121.stdout:smithi121 smithi121 2024-04-23T00:47:19.967 INFO:teuthology.orchestra.run.smithi121.stdout:smithi153 smithi153 2024-04-23T00:47:20.614 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-23T00:47:20.875 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:19.763425+0000 mgr.smithi121.otezso (mgr.14162) 246 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:21.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:19.763425+0000 mgr.smithi121.otezso (mgr.14162) 246 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:22.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:19.969282+0000 mgr.smithi121.otezso (mgr.14162) 247 : audit [DBG] from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:22.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:21.763863+0000 mgr.smithi121.otezso (mgr.14162) 248 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:22.256 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:19.969282+0000 mgr.smithi121.otezso (mgr.14162) 247 : audit [DBG] from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:22.256 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:21.763863+0000 mgr.smithi121.otezso (mgr.14162) 248 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:22.645 INFO:teuthology.orchestra.run.smithi121.stdout:Hostname Path Type Serial Size Health Ident Fault Available 2024-04-23T00:47:22.645 INFO:teuthology.orchestra.run.smithi121.stdout:smithi121 /dev/nvme0n1 ssd PHFT620400ZF400BGN 400G Unknown N/A N/A No 2024-04-23T00:47:22.645 INFO:teuthology.orchestra.run.smithi121.stdout:smithi153 /dev/nvme0n1 ssd CVFT623300MV400BGN 400G Unknown N/A N/A No 2024-04-23T00:47:22.998 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:22.646429+0000 mgr.smithi121.otezso (mgr.14162) 249 : audit [DBG] from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:23.313 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-23T00:47:23.325 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:23.326 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-23T00:47:23.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:22.646429+0000 mgr.smithi121.otezso (mgr.14162) 249 : audit [DBG] from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:23.355 INFO:teuthology.orchestra.run.smithi121.stderr:+ systemctl stop nfs-server 2024-04-23T00:47:23.363 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi153.front.sepia.ceph.com 2024-04-23T00:47:23.363 DEBUG:teuthology.orchestra.run.smithi153:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-23T00:47:23.400 INFO:teuthology.orchestra.run.smithi153.stderr:+ systemctl stop nfs-server 2024-04-23T00:47:23.410 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-23T00:47:23.420 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:23.420 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-23T00:47:24.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:23.764259+0000 mgr.smithi121.otezso (mgr.14162) 250 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:24.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:23.764259+0000 mgr.smithi121.otezso (mgr.14162) 250 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:26.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:25.490850+0000 mon.smithi121 (mon.0) 526 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-23T00:47:26.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:25.490850+0000 mon.smithi121 (mon.0) 526 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-23T00:47:27.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T00:47:26.776+0000 7ff4a8bf7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T00:47:27.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:25.490308+0000 mgr.smithi121.otezso (mgr.14162) 251 : audit [DBG] from='client.14496 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:27.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:25.764530+0000 mgr.smithi121.otezso (mgr.14162) 252 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:27.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:25.772473+0000 mon.smithi121 (mon.0) 527 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:25.772526+0000 mon.smithi121 (mon.0) 528 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:25.773611+0000 mon.smithi121 (mon.0) 529 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.008913+0000 mon.smithi121 (mon.0) 530 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.009060+0000 mon.smithi121 (mon.0) 531 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.773616+0000 mon.smithi121 (mon.0) 532 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:26.773798+0000 mon.smithi121 (mon.0) 533 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.775544+0000 mon.smithi121 (mon.0) 534 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:26.776303+0000 mon.smithi121 (mon.0) 535 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:26.776332+0000 mon.smithi121 (mon.0) 536 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:26.789525+0000 mon.smithi121 (mon.0) 537 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.789749+0000 mon.smithi121 (mon.0) 538 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:26.789870+0000 mon.smithi121 (mon.0) 539 : cluster [DBG] fsmap foofs:0 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:26.794060+0000 mgr.smithi121.otezso (mgr.14162) 253 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.797727+0000 mon.smithi121 (mon.0) 540 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.798484+0000 mon.smithi121 (mon.0) 541 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:27.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.808749+0000 mon.smithi121 (mon.0) 542 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.815505+0000 mon.smithi121 (mon.0) 543 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.816126+0000 mon.smithi121 (mon.0) 544 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T00:47:27.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.819009+0000 mon.smithi121 (mon.0) 545 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-23T00:47:27.105 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:26.819611+0000 mon.smithi121 (mon.0) 546 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:27.273 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:25.490308+0000 mgr.smithi121.otezso (mgr.14162) 251 : audit [DBG] from='client.14496 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:25.764530+0000 mgr.smithi121.otezso (mgr.14162) 252 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:25.772473+0000 mon.smithi121 (mon.0) 527 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:25.772526+0000 mon.smithi121 (mon.0) 528 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:25.773611+0000 mon.smithi121 (mon.0) 529 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.008913+0000 mon.smithi121 (mon.0) 530 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.009060+0000 mon.smithi121 (mon.0) 531 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.773616+0000 mon.smithi121 (mon.0) 532 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:26.773798+0000 mon.smithi121 (mon.0) 533 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.775544+0000 mon.smithi121 (mon.0) 534 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:26.776303+0000 mon.smithi121 (mon.0) 535 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T00:47:27.274 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:26.776332+0000 mon.smithi121 (mon.0) 536 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:26.789525+0000 mon.smithi121 (mon.0) 537 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.789749+0000 mon.smithi121 (mon.0) 538 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:26.789870+0000 mon.smithi121 (mon.0) 539 : cluster [DBG] fsmap foofs:0 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:26.794060+0000 mgr.smithi121.otezso (mgr.14162) 253 : cephadm [INF] Saving service mds.foofs spec with placement count:2 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.797727+0000 mon.smithi121 (mon.0) 540 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.798484+0000 mon.smithi121 (mon.0) 541 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.808749+0000 mon.smithi121 (mon.0) 542 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.815505+0000 mon.smithi121 (mon.0) 543 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.816126+0000 mon.smithi121 (mon.0) 544 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.819009+0000 mon.smithi121 (mon.0) 545 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-23T00:47:27.275 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:26.819611+0000 mon.smithi121 (mon.0) 546 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:27.417 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-23T00:47:27.426 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-23T00:47:27.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-23T00:47:28.812 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:26.820322+0000 mgr.smithi121.otezso (mgr.14162) 254 : cephadm [INF] Deploying daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T00:47:28.813 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:27.764979+0000 mgr.smithi121.otezso (mgr.14162) 255 : cluster [DBG] pgmap v224: 65 pgs: 11 creating+peering, 49 unknown, 5 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:28.813 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:27.789838+0000 mon.smithi121 (mon.0) 547 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-04-23T00:47:29.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:26.820322+0000 mgr.smithi121.otezso (mgr.14162) 254 : cephadm [INF] Deploying daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T00:47:29.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:27.764979+0000 mgr.smithi121.otezso (mgr.14162) 255 : cluster [DBG] pgmap v224: 65 pgs: 11 creating+peering, 49 unknown, 5 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:29.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:27.789838+0000 mon.smithi121 (mon.0) 547 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-04-23T00:47:29.473 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:29.473 INFO:teuthology.orchestra.run.smithi121.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"container_image_id": "0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0", "container_image_name": "docker.io/prom/alertmanager:v0.20.0", "created": "2024-04-23T00:41:15.984619Z", "last_refresh": "2024-04-23T00:45:43.430804Z", "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", "created": "2024-04-23T00:41:02.841325Z", "last_refresh": "2024-04-23T00:45:43.430861Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"container_image_id": "557c83e11646f123a27b5e4b62ac6c45e7bb8b2e90d6044034d0db5b7019415c", "container_image_name": "docker.io/ceph/ceph-grafana:6.7.4", "created": "2024-04-23T00:41:12.995137Z", "last_refresh": "2024-04-23T00:45:43.430917Z", "running": 1, "size": 1}}, {"events": ["2024-04-23T00:47:26.797873Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-23T00:47:26.794077Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "mix", "created": "2024-04-23T00:41:00.650258Z", "last_refresh": "2024-04-23T00:45:43.430741Z", "running": 2, "size": 2}}, {"events": ["2024-04-23T00:41:49.484649Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi121:172.21.15.121=smithi121", "smithi153:172.21.15.153=smithi153"]}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "mix", "created": "2024-04-23T00:41:49.481977Z", "last_refresh": "2024-04-23T00:45:43.430613Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"container_image_id": "e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87", "container_image_name": "docker.io/prom/node-exporter:v0.18.1", "created": "2024-04-23T00:41:14.603199Z", "last_refresh": "2024-04-23T00:45:43.430972Z", "running": 2, "size": 2}}, {"placement": {"hosts": ["smithi121"]}, "service_id": "None", "service_name": "osd.None", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", "last_refresh": "2024-04-23T00:45:43.431080Z", "running": 7, "size": 0}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"container_image_id": "de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490", "container_image_name": "docker.io/prom/prometheus:v2.18.1", "created": "2024-04-23T00:41:10.868310Z", "last_refresh": "2024-04-23T00:45:43.431027Z", "running": 1, "size": 1}}] 2024-04-23T00:47:30.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:28.794588+0000 mon.smithi121 (mon.0) 548 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-04-23T00:47:30.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.133250+0000 mon.smithi121 (mon.0) 549 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:30.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.133931+0000 mon.smithi121 (mon.0) 550 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T00:47:30.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.138719+0000 mon.smithi121 (mon.0) 551 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-23T00:47:30.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.139301+0000 mon.smithi121 (mon.0) 552 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:30.116 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:28.794588+0000 mon.smithi121 (mon.0) 548 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-04-23T00:47:30.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.133250+0000 mon.smithi121 (mon.0) 549 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:30.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.133931+0000 mon.smithi121 (mon.0) 550 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T00:47:30.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.138719+0000 mon.smithi121 (mon.0) 551 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-23T00:47:30.117 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.139301+0000 mon.smithi121 (mon.0) 552 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:30.457 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-23T00:47:31.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:29.139884+0000 mgr.smithi121.otezso (mgr.14162) 256 : cephadm [INF] Deploying daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T00:47:31.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.471899+0000 mgr.smithi121.otezso (mgr.14162) 257 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:31.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.765409+0000 mgr.smithi121.otezso (mgr.14162) 258 : cluster [DBG] pgmap v227: 65 pgs: 14 creating+peering, 23 unknown, 28 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:31.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798430+0000 mon.smithi121 (mon.0) 553 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:boot 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798488+0000 mon.smithi121 (mon.0) 554 : cluster [INF] daemon mds.foofs.smithi153.uunfav assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798554+0000 mon.smithi121 (mon.0) 555 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798574+0000 mon.smithi121 (mon.0) 556 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798584+0000 mon.smithi121 (mon.0) 557 : cluster [INF] Cluster is now healthy 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.798741+0000 mon.smithi121 (mon.0) 558 : cluster [DBG] fsmap foofs:0 1 up:standby 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:29.798807+0000 mon.smithi121 (mon.0) 559 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.801007+0000 mon.smithi121 (mon.0) 560 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:creating} 2024-04-23T00:47:31.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:29.811461+0000 mon.smithi121 (mon.0) 561 : cluster [INF] daemon mds.foofs.smithi153.uunfav is now active in filesystem foofs as rank 0 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:29.139884+0000 mgr.smithi121.otezso (mgr.14162) 256 : cephadm [INF] Deploying daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.471899+0000 mgr.smithi121.otezso (mgr.14162) 257 : audit [DBG] from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.765409+0000 mgr.smithi121.otezso (mgr.14162) 258 : cluster [DBG] pgmap v227: 65 pgs: 14 creating+peering, 23 unknown, 28 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798430+0000 mon.smithi121 (mon.0) 553 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:boot 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798488+0000 mon.smithi121 (mon.0) 554 : cluster [INF] daemon mds.foofs.smithi153.uunfav assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798554+0000 mon.smithi121 (mon.0) 555 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-23T00:47:31.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798574+0000 mon.smithi121 (mon.0) 556 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-23T00:47:31.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798584+0000 mon.smithi121 (mon.0) 557 : cluster [INF] Cluster is now healthy 2024-04-23T00:47:31.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.798741+0000 mon.smithi121 (mon.0) 558 : cluster [DBG] fsmap foofs:0 1 up:standby 2024-04-23T00:47:31.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:29.798807+0000 mon.smithi121 (mon.0) 559 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:47:31.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.801007+0000 mon.smithi121 (mon.0) 560 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:creating} 2024-04-23T00:47:31.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:29.811461+0000 mon.smithi121 (mon.0) 561 : cluster [INF] daemon mds.foofs.smithi153.uunfav is now active in filesystem foofs as rank 0 2024-04-23T00:47:31.458 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-23T00:47:32.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:30.810379+0000 mon.smithi121 (mon.0) 562 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:47:32.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:30.810444+0000 mon.smithi121 (mon.0) 563 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 2024-04-23T00:47:32.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:30.810379+0000 mon.smithi121 (mon.0) 562 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:47:32.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:30.810444+0000 mon.smithi121 (mon.0) 563 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 2024-04-23T00:47:33.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:31.765791+0000 mgr.smithi121.otezso (mgr.14162) 259 : cluster [DBG] pgmap v228: 65 pgs: 3 creating+peering, 62 active+clean; 1.5 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:33.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:32.376818+0000 mon.smithi121 (mon.0) 564 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:33.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:32.377451+0000 mon.smithi121 (mon.0) 565 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:33.079 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:31.765791+0000 mgr.smithi121.otezso (mgr.14162) 259 : cluster [DBG] pgmap v228: 65 pgs: 3 creating+peering, 62 active+clean; 1.5 KiB data, 44 MiB used, 715 GiB / 715 GiB avail 2024-04-23T00:47:33.080 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:32.376818+0000 mon.smithi121 (mon.0) 564 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:33.080 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:32.377451+0000 mon.smithi121 (mon.0) 565 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:33.789 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:33.789 INFO:teuthology.orchestra.run.smithi121.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"container_image_id": "0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0", "container_image_name": "docker.io/prom/alertmanager:v0.20.0", "created": "2024-04-23T00:41:15.984619Z", "last_refresh": "2024-04-23T00:45:43.430804Z", "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", "created": "2024-04-23T00:41:02.841325Z", "last_refresh": "2024-04-23T00:45:43.430861Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"container_image_id": "557c83e11646f123a27b5e4b62ac6c45e7bb8b2e90d6044034d0db5b7019415c", "container_image_name": "docker.io/ceph/ceph-grafana:6.7.4", "created": "2024-04-23T00:41:12.995137Z", "last_refresh": "2024-04-23T00:45:43.430917Z", "running": 1, "size": 1}}, {"events": ["2024-04-23T00:47:26.797873Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-23T00:47:26.794077Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "mix", "created": "2024-04-23T00:41:00.650258Z", "last_refresh": "2024-04-23T00:45:43.430741Z", "running": 2, "size": 2}}, {"events": ["2024-04-23T00:41:49.484649Z service:mon [INFO] \"service was created\""], "placement": {"count": 2, "hosts": ["smithi121:172.21.15.121=smithi121", "smithi153:172.21.15.153=smithi153"]}, "service_name": "mon", "service_type": "mon", "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "mix", "created": "2024-04-23T00:41:49.481977Z", "last_refresh": "2024-04-23T00:45:43.430613Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"container_image_id": "e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87", "container_image_name": "docker.io/prom/node-exporter:v0.18.1", "created": "2024-04-23T00:41:14.603199Z", "last_refresh": "2024-04-23T00:45:43.430972Z", "running": 2, "size": 2}}, {"placement": {"hosts": ["smithi121"]}, "service_id": "None", "service_name": "osd.None", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", "container_image_name": "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", "last_refresh": "2024-04-23T00:45:43.431080Z", "running": 7, "size": 0}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"container_image_id": "de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490", "container_image_name": "docker.io/prom/prometheus:v2.18.1", "created": "2024-04-23T00:41:10.868310Z", "last_refresh": "2024-04-23T00:45:43.431027Z", "running": 1, "size": 1}}] 2024-04-23T00:47:34.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:32.820615+0000 mon.smithi121 (mon.0) 566 : cluster [DBG] mds.? [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] up:boot 2024-04-23T00:47:34.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:32.820838+0000 mon.smithi121 (mon.0) 567 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:34.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:32.820918+0000 mon.smithi121 (mon.0) 568 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:47:34.064 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:32.826703+0000 mon.smithi121 (mon.0) 569 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:34.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:32.820615+0000 mon.smithi121 (mon.0) 566 : cluster [DBG] mds.? [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] up:boot 2024-04-23T00:47:34.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:32.820838+0000 mon.smithi121 (mon.0) 567 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:34.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:32.820918+0000 mon.smithi121 (mon.0) 568 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:47:34.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:32.826703+0000 mon.smithi121 (mon.0) 569 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:34.655 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-23T00:47:34.656 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-23T00:47:34.667 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:34.667 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-23T00:47:35.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:33.766189+0000 mgr.smithi121.otezso (mgr.14162) 260 : cluster [DBG] pgmap v229: 65 pgs: 65 active+clean; 1.7 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 9 op/s 2024-04-23T00:47:35.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:33.787511+0000 mgr.smithi121.otezso (mgr.14162) 261 : audit [DBG] from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:35.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:34.455621+0000 mon.smithi121 (mon.0) 570 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:35.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:33.766189+0000 mgr.smithi121.otezso (mgr.14162) 260 : cluster [DBG] pgmap v229: 65 pgs: 65 active+clean; 1.7 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 9 op/s 2024-04-23T00:47:35.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:33.787511+0000 mgr.smithi121.otezso (mgr.14162) 261 : audit [DBG] from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T00:47:35.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:34.455621+0000 mon.smithi121 (mon.0) 570 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:36.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:35.766678+0000 mgr.smithi121.otezso (mgr.14162) 262 : cluster [DBG] pgmap v230: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 6 op/s 2024-04-23T00:47:36.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:35.766678+0000 mgr.smithi121.otezso (mgr.14162) 262 : cluster [DBG] pgmap v230: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 6 op/s 2024-04-23T00:47:37.557 INFO:teuthology.orchestra.run.smithi121.stderr:Invalid command: missing required parameter clusterid() 2024-04-23T00:47:37.557 INFO:teuthology.orchestra.run.smithi121.stderr:nfs cluster create [] : Create an NFS Cluster 2024-04-23T00:47:37.558 INFO:teuthology.orchestra.run.smithi121.stderr:Error EINVAL: invalid command 2024-04-23T00:47:38.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:36.829760+0000 mon.smithi121 (mon.0) 571 : cluster [DBG] mds.? [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] up:standby 2024-04-23T00:47:38.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:36.829809+0000 mon.smithi121 (mon.0) 572 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:38.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:37.767061+0000 mgr.smithi121.otezso (mgr.14162) 263 : cluster [DBG] pgmap v231: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-23T00:47:38.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:36.829760+0000 mon.smithi121 (mon.0) 571 : cluster [DBG] mds.? [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] up:standby 2024-04-23T00:47:38.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:36.829809+0000 mon.smithi121 (mon.0) 572 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:47:38.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:37.767061+0000 mgr.smithi121.otezso (mgr.14162) 263 : cluster [DBG] pgmap v231: 65 pgs: 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-23T00:47:39.078 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:37.873475+0000 mgr.smithi121.otezso (mgr.14162) 264 : audit [DBG] from='client.14520 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "type": "cephfs", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:39.078 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:37.874377+0000 mon.smithi121 (mon.0) 573 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2024-04-23T00:47:39.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:37.873475+0000 mgr.smithi121.otezso (mgr.14162) 264 : audit [DBG] from='client.14520 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "type": "cephfs", "clusterid": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:39.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:37.874377+0000 mon.smithi121 (mon.0) 573 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]: dispatch 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:38.839813+0000 mon.smithi121 (mon.0) 574 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:38.839867+0000 mon.smithi121 (mon.0) 575 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:38.840953+0000 mon.smithi121 (mon.0) 576 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.739331+0000 mon.smithi121 (mon.0) 577 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.742157+0000 mon.smithi121 (mon.0) 578 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:39.848 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.744701+0000 mon.smithi121 (mon.0) 579 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:39.849 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.748082+0000 mon.smithi121 (mon.0) 580 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:39.849 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:39.767527+0000 mgr.smithi121.otezso (mgr.14162) 265 : cluster [DBG] pgmap v233: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-23T00:47:39.861 INFO:teuthology.orchestra.run.smithi121.stdout:NFS Cluster Created Successfully 2024-04-23T00:47:40.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:38.839813+0000 mon.smithi121 (mon.0) 574 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool create", "pool": "nfs-ganesha"}]': finished 2024-04-23T00:47:40.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:38.839867+0000 mon.smithi121 (mon.0) 575 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-04-23T00:47:40.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:38.840953+0000 mon.smithi121 (mon.0) 576 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]: dispatch 2024-04-23T00:47:40.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.739331+0000 mon.smithi121 (mon.0) 577 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:40.322 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.742157+0000 mon.smithi121 (mon.0) 578 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:40.322 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.744701+0000 mon.smithi121 (mon.0) 579 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:40.322 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.748082+0000 mon.smithi121 (mon.0) 580 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:40.322 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:39.767527+0000 mgr.smithi121.otezso (mgr.14162) 265 : cluster [DBG] pgmap v233: 97 pgs: 5 creating+peering, 27 unknown, 65 active+clean; 2.3 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-23T00:47:40.570 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-23T00:47:40.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.841092+0000 mon.smithi121 (mon.0) 581 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2024-04-23T00:47:40.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:39.841281+0000 mon.smithi121 (mon.0) 582 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.841092+0000 mon.smithi121 (mon.0) 581 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "pool": "nfs-ganesha", "app": "nfs"}]': finished 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:39.841281+0000 mon.smithi121 (mon.0) 582 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.860203+0000 mgr.smithi121.otezso (mgr.14162) 266 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.863255+0000 mon.smithi121 (mon.0) 583 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.863773+0000 mon.smithi121 (mon.0) 584 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.869526+0000 mon.smithi121 (mon.0) 585 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:41.103 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.872132+0000 mgr.smithi121.otezso (mgr.14162) 267 : cephadm [INF] Checking pool "nfs-ganesha" exists for service nfs.foo 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.878620+0000 mgr.smithi121.otezso (mgr.14162) 268 : cephadm [INF] Creating key for client.nfs.foo.smithi153 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.878867+0000 mon.smithi121 (mon.0) 586 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.881954+0000 mon.smithi121 (mon.0) 587 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.883165+0000 mgr.smithi121.otezso (mgr.14162) 269 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.883233+0000 mgr.smithi121.otezso (mgr.14162) 270 : cephadm [INF] Creating key for client.nfs.foo.smithi153-rgw 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.883427+0000 mon.smithi121 (mon.0) 588 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.886314+0000 mon.smithi121 (mon.0) 589 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:39.888642+0000 mon.smithi121 (mon.0) 590 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:41.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:39.889251+0000 mgr.smithi121.otezso (mgr.14162) 271 : cephadm [INF] Deploying daemon nfs.foo.smithi153 on smithi153 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.860203+0000 mgr.smithi121.otezso (mgr.14162) 266 : cephadm [INF] Saving service nfs.foo spec with placement count:2 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.863255+0000 mon.smithi121 (mon.0) 583 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.863773+0000 mon.smithi121 (mon.0) 584 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.869526+0000 mon.smithi121 (mon.0) 585 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.872132+0000 mgr.smithi121.otezso (mgr.14162) 267 : cephadm [INF] Checking pool "nfs-ganesha" exists for service nfs.foo 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.878620+0000 mgr.smithi121.otezso (mgr.14162) 268 : cephadm [INF] Creating key for client.nfs.foo.smithi153 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.878867+0000 mon.smithi121 (mon.0) 586 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.881954+0000 mon.smithi121 (mon.0) 587 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.883165+0000 mgr.smithi121.otezso (mgr.14162) 269 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:47:41.312 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.883233+0000 mgr.smithi121.otezso (mgr.14162) 270 : cephadm [INF] Creating key for client.nfs.foo.smithi153-rgw 2024-04-23T00:47:41.313 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.883427+0000 mon.smithi121 (mon.0) 588 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:47:41.313 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.886314+0000 mon.smithi121 (mon.0) 589 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi153-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:47:41.313 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:39.888642+0000 mon.smithi121 (mon.0) 590 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:41.313 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:39.889251+0000 mgr.smithi121.otezso (mgr.14162) 271 : cephadm [INF] Deploying daemon nfs.foo.smithi153 on smithi153 2024-04-23T00:47:42.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:40.850270+0000 mon.smithi121 (mon.0) 591 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-04-23T00:47:42.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:41.767915+0000 mgr.smithi121.otezso (mgr.14162) 272 : cluster [DBG] pgmap v236: 97 pgs: 2 creating+activating, 14 creating+peering, 13 unknown, 68 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-23T00:47:42.191 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:40.850270+0000 mon.smithi121 (mon.0) 591 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-04-23T00:47:42.192 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:41.767915+0000 mgr.smithi121.otezso (mgr.14162) 272 : cluster [DBG] pgmap v236: 97 pgs: 2 creating+activating, 14 creating+peering, 13 unknown, 68 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-23T00:47:42.766 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:47:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "bind": "/fake", 2024-04-23T00:47:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "fs": "foofs", 2024-04-23T00:47:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "path": "/", 2024-04-23T00:47:42.766 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster": "foo", 2024-04-23T00:47:42.767 INFO:teuthology.orchestra.run.smithi121.stdout: "mode": "RW" 2024-04-23T00:47:42.767 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:47:43.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:41.864663+0000 mon.smithi121 (mon.0) 592 : cluster [DBG] mgrmap e19: smithi121.otezso(active, since 6m), standbys: smithi153.olhmgd 2024-04-23T00:47:43.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:42.743025+0000 mgr.smithi121.otezso (mgr.14162) 273 : audit [DBG] from='client.14532 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:43.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:42.760032+0000 mon.smithi121 (mon.0) 593 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:47:43.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:42.764034+0000 mon.smithi121 (mon.0) 594 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-23T00:47:43.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:41.864663+0000 mon.smithi121 (mon.0) 592 : cluster [DBG] mgrmap e19: smithi121.otezso(active, since 6m), standbys: smithi153.olhmgd 2024-04-23T00:47:43.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:42.743025+0000 mgr.smithi121.otezso (mgr.14162) 273 : audit [DBG] from='client.14532 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "clusterid": "foo", "binding": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:43.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:42.760032+0000 mon.smithi121 (mon.0) 593 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:47:43.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:42.764034+0000 mon.smithi121 (mon.0) 594 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.foo1", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-23T00:47:43.488 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-23T00:47:44.972 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:43.768344+0000 mgr.smithi121.otezso (mgr.14162) 274 : cluster [DBG] pgmap v237: 97 pgs: 2 creating+activating, 14 creating+peering, 81 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T00:47:44.972 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.800020+0000 mon.smithi121 (mon.0) 595 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:44.972 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:43.800661+0000 mgr.smithi121.otezso (mgr.14162) 275 : cephadm [INF] Creating key for client.nfs.foo.smithi121 2024-04-23T00:47:44.972 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.800872+0000 mon.smithi121 (mon.0) 596 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-04-23T00:47:44.972 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.804079+0000 mon.smithi121 (mon.0) 597 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:43.805129+0000 mgr.smithi121.otezso (mgr.14162) 276 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:43.805190+0000 mgr.smithi121.otezso (mgr.14162) 277 : cephadm [INF] Creating key for client.nfs.foo.smithi121-rgw 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.805390+0000 mon.smithi121 (mon.0) 598 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.808468+0000 mon.smithi121 (mon.0) 599 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:43.809279+0000 mon.smithi121 (mon.0) 600 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:44.973 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:47:43.809887+0000 mgr.smithi121.otezso (mgr.14162) 278 : cephadm [INF] Deploying daemon nfs.foo.smithi121 on smithi121 2024-04-23T00:47:45.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:43.768344+0000 mgr.smithi121.otezso (mgr.14162) 274 : cluster [DBG] pgmap v237: 97 pgs: 2 creating+activating, 14 creating+peering, 81 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.800020+0000 mon.smithi121 (mon.0) 595 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:43.800661+0000 mgr.smithi121.otezso (mgr.14162) 275 : cephadm [INF] Creating key for client.nfs.foo.smithi121 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.800872+0000 mon.smithi121 (mon.0) 596 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]: dispatch 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.804079+0000 mon.smithi121 (mon.0) 597 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121", "caps": ["mon", "allow r", "osd", "allow rw pool=nfs-ganesha namespace=foo"]}]': finished 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:43.805129+0000 mgr.smithi121.otezso (mgr.14162) 276 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:43.805190+0000 mgr.smithi121.otezso (mgr.14162) 277 : cephadm [INF] Creating key for client.nfs.foo.smithi121-rgw 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.805390+0000 mon.smithi121 (mon.0) 598 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.808468+0000 mon.smithi121 (mon.0) 599 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.smithi121-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:43.809279+0000 mon.smithi121 (mon.0) 600 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:47:45.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:47:43.809887+0000 mgr.smithi121.otezso (mgr.14162) 278 : cephadm [INF] Deploying daemon nfs.foo.smithi121 on smithi121 2024-04-23T00:47:46.876 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:45.768995+0000 mgr.smithi121.otezso (mgr.14162) 279 : cluster [DBG] pgmap v238: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 295 B/s rd, 591 B/s wr, 1 op/s 2024-04-23T00:47:47.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:45.768995+0000 mgr.smithi121.otezso (mgr.14162) 279 : cluster [DBG] pgmap v238: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 295 B/s rd, 591 B/s wr, 1 op/s 2024-04-23T00:47:48.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:46.383265+0000 mgr.smithi121.otezso (mgr.14162) 280 : audit [DBG] from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:48.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:46.383265+0000 mgr.smithi121.otezso (mgr.14162) 280 : audit [DBG] from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:49.017 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo 2/2 - 9s count:2 2024-04-23T00:47:49.031 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:47.682144+0000 mgr.smithi121.otezso (mgr.14162) 281 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:49.031 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:47.769490+0000 mgr.smithi121.otezso (mgr.14162) 282 : cluster [DBG] pgmap v239: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T00:47:49.075 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:47.682144+0000 mgr.smithi121.otezso (mgr.14162) 281 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:49.075 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:47.769490+0000 mgr.smithi121.otezso (mgr.14162) 282 : cluster [DBG] pgmap v239: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T00:47:49.740 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-23T00:47:49.754 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:49.755 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-23T00:47:49.786 INFO:teuthology.orchestra.run.smithi121.stderr:+ mkdir /mnt/foo 2024-04-23T00:47:49.798 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-23T00:47:49.865 INFO:teuthology.orchestra.run.smithi121.stderr:++ hostname 2024-04-23T00:47:49.866 INFO:teuthology.orchestra.run.smithi121.stderr:+ mount -t nfs smithi121:/fake /mnt/foo -o sync 2024-04-23T00:47:49.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:48.859514+0000 mon.smithi121 (mon.0) 601 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:49.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:48.860196+0000 mon.smithi121 (mon.0) 602 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:49.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:49.001469+0000 mgr.smithi121.otezso (mgr.14162) 283 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:49.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:49.769923+0000 mgr.smithi121.otezso (mgr.14162) 284 : cluster [DBG] pgmap v240: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-23T00:47:50.070 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:48.859514+0000 mon.smithi121 (mon.0) 601 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:50.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:48.860196+0000 mon.smithi121 (mon.0) 602 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:47:50.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:49.001469+0000 mgr.smithi121.otezso (mgr.14162) 283 : audit [DBG] from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:47:50.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:49.769923+0000 mgr.smithi121.otezso (mgr.14162) 284 : cluster [DBG] pgmap v240: 97 pgs: 97 active+clean; 2.7 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-23T00:47:50.576 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-23T00:47:50.647 INFO:teuthology.orchestra.run.smithi121.stderr:+ echo test 2024-04-23T00:47:50.656 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-23T00:47:50.724 INFO:teuthology.orchestra.run.smithi121.stderr:+ sync 2024-04-23T00:47:51.433 INFO:teuthology.run_tasks:Running task parallel... 2024-04-23T00:47:51.447 INFO:teuthology.task.parallel:starting parallel... 2024-04-23T00:47:51.448 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-23T00:47:51.448 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-23T00:47:51.449 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:51.449 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-23T00:47:51.449 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-23T00:47:51.449 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-23T00:47:51.450 INFO:teuthology.task.exec:Executing custom commands... 2024-04-23T00:47:51.450 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:47:51.450 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-23T00:47:51.539 INFO:teuthology.orchestra.run.smithi121.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-23T00:47:51.539 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:47:51.540 INFO:teuthology.orchestra.run.smithi121.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-23T00:47:51.549 INFO:teuthology.orchestra.run.smithi121.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-23T00:47:51.650 INFO:teuthology.orchestra.run.smithi121.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-23T00:47:51.650 INFO:teuthology.orchestra.run.smithi121.stdout:releasing clients 2024-04-23T00:47:52.650 INFO:teuthology.orchestra.run.smithi121.stdout: 5 315 57.43 MB/sec warmup 1 sec latency 12.246 ms 2024-04-23T00:47:53.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:51.682266+0000 mon.smithi121 (mon.0) 603 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:53.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:51.770336+0000 mgr.smithi121.otezso (mgr.14162) 285 : cluster [DBG] pgmap v241: 97 pgs: 97 active+clean; 4.1 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-23T00:47:53.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:51.682266+0000 mon.smithi121 (mon.0) 603 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:47:53.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:51.770336+0000 mgr.smithi121.otezso (mgr.14162) 285 : cluster [DBG] pgmap v241: 97 pgs: 97 active+clean; 4.1 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-23T00:47:53.650 INFO:teuthology.orchestra.run.smithi121.stdout: 5 754 53.35 MB/sec warmup 2 sec latency 11.386 ms 2024-04-23T00:47:54.348 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:53.770791+0000 mgr.smithi121.otezso (mgr.14162) 286 : cluster [DBG] pgmap v242: 97 pgs: 97 active+clean; 27 MiB data, 89 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.2 MiB/s wr, 94 op/s 2024-04-23T00:47:54.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:53.770791+0000 mgr.smithi121.otezso (mgr.14162) 286 : cluster [DBG] pgmap v242: 97 pgs: 97 active+clean; 27 MiB data, 89 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.2 MiB/s wr, 94 op/s 2024-04-23T00:47:54.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 2213 44.78 MB/sec warmup 3 sec latency 11.910 ms 2024-04-23T00:47:55.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 3683 48.42 MB/sec warmup 4 sec latency 9.275 ms 2024-04-23T00:47:55.864 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:54.863543+0000 mon.smithi121 (mon.0) 604 : audit [INF] from='client.? 172.21.15.121:0/732504971' entity='client.admin' 2024-04-23T00:47:55.864 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:55.771308+0000 mgr.smithi121.otezso (mgr.14162) 287 : cluster [DBG] pgmap v243: 97 pgs: 97 active+clean; 71 MiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 6.2 MiB/s wr, 310 op/s 2024-04-23T00:47:56.037 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-23T00:47:56.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:54.863543+0000 mon.smithi121 (mon.0) 604 : audit [INF] from='client.? 172.21.15.121:0/732504971' entity='client.admin' 2024-04-23T00:47:56.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:55.771308+0000 mgr.smithi121.otezso (mgr.14162) 287 : cluster [DBG] pgmap v243: 97 pgs: 97 active+clean; 71 MiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 6.2 MiB/s wr, 310 op/s 2024-04-23T00:47:56.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 4664 42.06 MB/sec warmup 5 sec latency 18.514 ms 2024-04-23T00:47:57.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 6055 39.77 MB/sec warmup 6 sec latency 12.538 ms 2024-04-23T00:47:58.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 7363 42.10 MB/sec warmup 7 sec latency 13.954 ms 2024-04-23T00:47:59.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:57.771749+0000 mgr.smithi121.otezso (mgr.14162) 288 : cluster [DBG] pgmap v244: 97 pgs: 97 active+clean; 111 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 10 MiB/s wr, 516 op/s 2024-04-23T00:47:59.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:57.771749+0000 mgr.smithi121.otezso (mgr.14162) 288 : cluster [DBG] pgmap v244: 97 pgs: 97 active+clean; 111 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 10 MiB/s wr, 516 op/s 2024-04-23T00:47:59.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 8401 38.69 MB/sec warmup 8 sec latency 19.682 ms 2024-04-23T00:47:59.665 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-23T00:48:00.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:47:58.842137+0000 mon.smithi121 (mon.0) 605 : audit [INF] from='client.? 172.21.15.121:0/1988207236' entity='client.admin' 2024-04-23T00:48:00.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:47:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:47:59.772218+0000 mgr.smithi121.otezso (mgr.14162) 289 : cluster [DBG] pgmap v245: 97 pgs: 97 active+clean; 132 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 15 MiB/s wr, 884 op/s 2024-04-23T00:48:00.320 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:47:58.842137+0000 mon.smithi121 (mon.0) 605 : audit [INF] from='client.? 172.21.15.121:0/1988207236' entity='client.admin' 2024-04-23T00:48:00.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:47:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:47:59.772218+0000 mgr.smithi121.otezso (mgr.14162) 289 : cluster [DBG] pgmap v245: 97 pgs: 97 active+clean; 132 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 15 MiB/s wr, 884 op/s 2024-04-23T00:48:00.652 INFO:teuthology.orchestra.run.smithi121.stdout: 5 9693 37.62 MB/sec warmup 9 sec latency 17.496 ms 2024-04-23T00:48:01.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:00.117881+0000 mon.smithi121 (mon.0) 606 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:00.122552+0000 mon.smithi121 (mon.0) 607 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:00.126873+0000 mon.smithi121 (mon.0) 608 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:01.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:00.133205+0000 mon.smithi121 (mon.0) 609 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:00.117881+0000 mon.smithi121 (mon.0) 606 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:00.122552+0000 mon.smithi121 (mon.0) 607 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:00.126873+0000 mon.smithi121 (mon.0) 608 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:01.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:00.133205+0000 mon.smithi121 (mon.0) 609 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:01.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 11109 39.16 MB/sec warmup 10 sec latency 7.873 ms 2024-04-23T00:48:02.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:01.772714+0000 mgr.smithi121.otezso (mgr.14162) 290 : cluster [DBG] pgmap v246: 97 pgs: 97 active+clean; 149 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 19 MiB/s wr, 1.35k op/s 2024-04-23T00:48:02.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:01.772714+0000 mgr.smithi121.otezso (mgr.14162) 290 : cluster [DBG] pgmap v246: 97 pgs: 97 active+clean; 149 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 19 MiB/s wr, 1.35k op/s 2024-04-23T00:48:02.651 INFO:teuthology.orchestra.run.smithi121.stdout: 5 12343 37.99 MB/sec warmup 11 sec latency 10.466 ms 2024-04-23T00:48:03.228 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph mgr module enable nfs --force' 2024-04-23T00:48:03.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:02.447070+0000 mon.smithi121 (mon.0) 610 : audit [INF] from='client.? 172.21.15.121:0/2533295250' entity='client.admin' 2024-04-23T00:48:03.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:02.450985+0000 mon.smithi121 (mon.0) 611 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:03.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:02.460171+0000 mon.smithi121 (mon.0) 612 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:03.652 INFO:teuthology.orchestra.run.smithi121.stdout: 5 13734 38.04 MB/sec warmup 12 sec latency 13.680 ms 2024-04-23T00:48:03.820 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:02.447070+0000 mon.smithi121 (mon.0) 610 : audit [INF] from='client.? 172.21.15.121:0/2533295250' entity='client.admin' 2024-04-23T00:48:03.821 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:02.450985+0000 mon.smithi121 (mon.0) 611 : audit [DBG] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:03.821 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:02.460171+0000 mon.smithi121 (mon.0) 612 : audit [INF] from='mgr.14162 172.21.15.121:0/2536348369' entity='mgr.smithi121.otezso' 2024-04-23T00:48:04.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 14760 37.51 MB/sec warmup 13 sec latency 13.909 ms 2024-04-23T00:48:04.820 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:03.773159+0000 mgr.smithi121.otezso (mgr.14162) 291 : cluster [DBG] pgmap v247: 97 pgs: 97 active+clean; 149 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 21 MiB/s wr, 1.43k op/s 2024-04-23T00:48:04.875 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:03.773159+0000 mgr.smithi121.otezso (mgr.14162) 291 : cluster [DBG] pgmap v247: 97 pgs: 97 active+clean; 149 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 21 MiB/s wr, 1.43k op/s 2024-04-23T00:48:05.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 15802 36.65 MB/sec warmup 14 sec latency 12.866 ms 2024-04-23T00:48:06.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 17229 36.66 MB/sec warmup 15 sec latency 15.262 ms 2024-04-23T00:48:07.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:05.773704+0000 mgr.smithi121.otezso (mgr.14162) 292 : cluster [DBG] pgmap v248: 97 pgs: 97 active+clean; 153 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 21 MiB/s wr, 1.53k op/s 2024-04-23T00:48:07.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:05.880118+0000 mon.smithi121 (mon.0) 613 : audit [INF] from='client.? 172.21.15.121:0/1036313210' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-04-23T00:48:07.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:05.773704+0000 mgr.smithi121.otezso (mgr.14162) 292 : cluster [DBG] pgmap v248: 97 pgs: 97 active+clean; 153 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 21 MiB/s wr, 1.53k op/s 2024-04-23T00:48:07.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:05.880118+0000 mon.smithi121 (mon.0) 613 : audit [INF] from='client.? 172.21.15.121:0/1036313210' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]: dispatch 2024-04-23T00:48:07.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 18303 36.46 MB/sec warmup 16 sec latency 37.054 ms 2024-04-23T00:48:08.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:06.839260+0000 mon.smithi121 (mon.0) 614 : audit [INF] from='client.? 172.21.15.121:0/1036313210' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2024-04-23T00:48:08.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:06.839337+0000 mon.smithi121 (mon.0) 615 : cluster [DBG] mgrmap e20: smithi121.otezso(active, since 6m), standbys: smithi153.olhmgd 2024-04-23T00:48:08.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:06.839260+0000 mon.smithi121 (mon.0) 614 : audit [INF] from='client.? 172.21.15.121:0/1036313210' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": "--force"}]': finished 2024-04-23T00:48:08.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:06.839337+0000 mon.smithi121 (mon.0) 615 : cluster [DBG] mgrmap e20: smithi121.otezso(active, since 6m), standbys: smithi153.olhmgd 2024-04-23T00:48:08.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 19137 35.26 MB/sec warmup 17 sec latency 65.950 ms 2024-04-23T00:48:09.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 20291 34.61 MB/sec warmup 18 sec latency 41.823 ms 2024-04-23T00:48:10.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 21341 35.10 MB/sec warmup 19 sec latency 27.755 ms 2024-04-23T00:48:11.121 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-23T00:48:11.653 INFO:teuthology.orchestra.run.smithi121.stdout: 5 22129 34.08 MB/sec warmup 20 sec latency 13.753 ms 2024-04-23T00:48:12.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 23240 33.66 MB/sec warmup 21 sec latency 12.743 ms 2024-04-23T00:48:13.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 24382 33.73 MB/sec warmup 22 sec latency 30.427 ms 2024-04-23T00:48:14.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:13.396682+0000 mon.smithi121 (mon.0) 616 : cluster [DBG] Standby manager daemon smithi153.olhmgd restarted 2024-04-23T00:48:14.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:13.396799+0000 mon.smithi121 (mon.0) 617 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:13.397793+0000 mon.smithi153 (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:13.398228+0000 mon.smithi153 (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:13.403448+0000 mon.smithi153 (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:13.404119+0000 mon.smithi153 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:13.798540+0000 mon.smithi121 (mon.0) 618 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:13.798981+0000 mon.smithi121 (mon.0) 619 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:48:14.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:13.801737+0000 mon.smithi121 (mon.0) 620 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-04-23T00:48:14.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:13.396682+0000 mon.smithi121 (mon.0) 616 : cluster [DBG] Standby manager daemon smithi153.olhmgd restarted 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:13.396799+0000 mon.smithi121 (mon.0) 617 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:13.397793+0000 mon.smithi153 (mon.1) 18 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:13.398228+0000 mon.smithi153 (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:13.403448+0000 mon.smithi153 (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:13.404119+0000 mon.smithi153 (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.153:0/2209748662' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:13.798540+0000 mon.smithi121 (mon.0) 618 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:13.798981+0000 mon.smithi121 (mon.0) 619 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T00:48:14.437 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:13.801737+0000 mon.smithi121 (mon.0) 620 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-04-23T00:48:14.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 25408 33.67 MB/sec warmup 23 sec latency 16.323 ms 2024-04-23T00:48:15.149 INFO:teuthology.orchestra.run.smithi121.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:14.131844+0000 mon.smithi121 (mon.0) 621 : cluster [DBG] mgrmap e21: smithi121.otezso(active, starting, since 0.332949s), standbys: smithi153.olhmgd 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.133330+0000 mon.smithi153 (mon.1) 22 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.133570+0000 mon.smithi153 (mon.1) 23 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.134377+0000 mon.smithi153 (mon.1) 24 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.134603+0000 mon.smithi153 (mon.1) 25 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.134816+0000 mon.smithi153 (mon.1) 26 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.134966+0000 mon.smithi153 (mon.1) 27 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.135166+0000 mon.smithi153 (mon.1) 28 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:48:15.391 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.135373+0000 mon.smithi153 (mon.1) 29 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.135565+0000 mon.smithi153 (mon.1) 30 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.135740+0000 mon.smithi153 (mon.1) 31 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.135889+0000 mon.smithi153 (mon.1) 32 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.136035+0000 mon.smithi153 (mon.1) 33 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.136234+0000 mon.smithi153 (mon.1) 34 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.136419+0000 mon.smithi153 (mon.1) 35 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.136936+0000 mon.smithi153 (mon.1) 36 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.137187+0000 mon.smithi153 (mon.1) 37 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.137761+0000 mon.smithi153 (mon.1) 38 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:14.141831+0000 mon.smithi121 (mon.0) 622 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.286820+0000 mon.smithi153 (mon.1) 39 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.288580+0000 mon.smithi153 (mon.1) 40 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.315519+0000 mon.smithi153 (mon.1) 41 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.315894+0000 mon.smithi121 (mon.0) 623 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:48:15.392 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.320664+0000 mon.smithi153 (mon.1) 42 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:48:15.393 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:14.321363+0000 mon.smithi121 (mon.0) 624 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:48:15.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:14.131844+0000 mon.smithi121 (mon.0) 621 : cluster [DBG] mgrmap e21: smithi121.otezso(active, starting, since 0.332949s), standbys: smithi153.olhmgd 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.133330+0000 mon.smithi153 (mon.1) 22 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.133570+0000 mon.smithi153 (mon.1) 23 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.134377+0000 mon.smithi153 (mon.1) 24 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.134603+0000 mon.smithi153 (mon.1) 25 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.134816+0000 mon.smithi153 (mon.1) 26 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.134966+0000 mon.smithi153 (mon.1) 27 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.135166+0000 mon.smithi153 (mon.1) 28 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.135373+0000 mon.smithi153 (mon.1) 29 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.135565+0000 mon.smithi153 (mon.1) 30 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.135740+0000 mon.smithi153 (mon.1) 31 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.135889+0000 mon.smithi153 (mon.1) 32 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:48:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.136035+0000 mon.smithi153 (mon.1) 33 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.136234+0000 mon.smithi153 (mon.1) 34 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.136419+0000 mon.smithi153 (mon.1) 35 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.136936+0000 mon.smithi153 (mon.1) 36 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.137187+0000 mon.smithi153 (mon.1) 37 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.137761+0000 mon.smithi153 (mon.1) 38 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:14.141831+0000 mon.smithi121 (mon.0) 622 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.286820+0000 mon.smithi153 (mon.1) 39 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.288580+0000 mon.smithi153 (mon.1) 40 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.315519+0000 mon.smithi153 (mon.1) 41 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.315894+0000 mon.smithi121 (mon.0) 623 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.320664+0000 mon.smithi153 (mon.1) 42 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:48:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:14.321363+0000 mon.smithi121 (mon.0) 624 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:48:15.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 26318 33.08 MB/sec warmup 24 sec latency 16.444 ms 2024-04-23T00:48:16.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:15.140604+0000 mon.smithi121 (mon.0) 625 : cluster [DBG] mgrmap e22: smithi121.otezso(active, since 1.34171s), standbys: smithi153.olhmgd 2024-04-23T00:48:16.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:15.144990+0000 mgr.smithi121.otezso (mgr.14594) 2 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:48:16.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:15.145661+0000 mgr.smithi121.otezso (mgr.14594) 3 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:16.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:15.149051+0000 mon.smithi121 (mon.0) 626 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:16.556 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-23T00:48:16.557 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T00:48:16.557 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-23T00:48:16.570 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:15.140604+0000 mon.smithi121 (mon.0) 625 : cluster [DBG] mgrmap e22: smithi121.otezso(active, since 1.34171s), standbys: smithi153.olhmgd 2024-04-23T00:48:16.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:15.144990+0000 mgr.smithi121.otezso (mgr.14594) 2 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:48:16.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:15.145661+0000 mgr.smithi121.otezso (mgr.14594) 3 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:16.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:15.149051+0000 mon.smithi121 (mon.0) 626 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:16.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 27561 32.87 MB/sec warmup 25 sec latency 23.925 ms 2024-04-23T00:48:17.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:16.136421+0000 mgr.smithi121.otezso (mgr.14594) 4 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:17.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:16.154413+0000 mon.smithi121 (mon.0) 627 : cluster [DBG] mgrmap e23: smithi121.otezso(active, since 2s), standbys: smithi153.olhmgd 2024-04-23T00:48:17.477 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:16.136421+0000 mgr.smithi121.otezso (mgr.14594) 4 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:17.478 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:16.154413+0000 mon.smithi121 (mon.0) 627 : cluster [DBG] mgrmap e23: smithi121.otezso(active, since 2s), standbys: smithi153.olhmgd 2024-04-23T00:48:17.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 28739 33.40 MB/sec warmup 26 sec latency 25.832 ms 2024-04-23T00:48:18.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 29619 32.72 MB/sec warmup 27 sec latency 18.243 ms 2024-04-23T00:48:19.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:18.136735+0000 mgr.smithi121.otezso (mgr.14594) 5 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:19.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:18.289077+0000 mon.smithi121 (mon.0) 628 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:19.605 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:48:19.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:18.136735+0000 mgr.smithi121.otezso (mgr.14594) 5 : cluster [DBG] pgmap v5: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:19.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:18.289077+0000 mon.smithi121 (mon.0) 628 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:19.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 30741 32.29 MB/sec warmup 28 sec latency 21.016 ms 2024-04-23T00:48:20.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:19.592949+0000 mgr.smithi121.otezso (mgr.14594) 6 : audit [DBG] from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 running (4m) 20s ago 6m *:9093 *:9094 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (6m) 20s ago 6m - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (5m) 2s ago 5m - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 running (4m) 20s ago 6m *:3000 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (48s) 20s ago 48s - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:48:20.403 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (51s) 2s ago 51s - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 running (7m) 20s ago 7m *:9283 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 running (5m) 2s ago 5m *:8443 *:9283 16.2.0 24ecd6d5f14c 29976d00060b 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (7m) 20s ago 7m - 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (5m) 2s ago 5m - 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi121 smithi121 running (32s) 20s ago 31s *:2049 3.5 24ecd6d5f14c 6cbfa398c679 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi153 smithi153 running (37s) 2s ago 36s *:2049 3.5 24ecd6d5f14c 09a6f5208198 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 running (5m) 20s ago 5m *:9100 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 running (5m) 2s ago 5m *:9100 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (4m) 20s ago 4m - 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (4m) 20s ago 4m - 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (4m) 20s ago 3m - 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (3m) 20s ago 3m - 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (3m) 2s ago 3m - 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (3m) 2s ago 3m - 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:48:20.404 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (2m) 2s ago 2m - 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:48:20.405 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (2m) 2s ago 2m - 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:48:20.405 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 running (4m) 20s ago 5m *:9095 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:48:20.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:19.592949+0000 mgr.smithi121.otezso (mgr.14594) 6 : audit [DBG] from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:20.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 31767 32.61 MB/sec warmup 29 sec latency 22.213 ms 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:48:20.893 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 14 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:48:20.894 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:48:21.293 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:20.016068+0000 mgr.smithi121.otezso (mgr.14594) 7 : audit [DBG] from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:21.294 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:20.139576+0000 mgr.smithi121.otezso (mgr.14594) 8 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:21.294 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:20.394289+0000 mgr.smithi121.otezso (mgr.14594) 9 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:21.294 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:20.894086+0000 mon.smithi153 (mon.1) 43 : audit [DBG] from='client.? 172.21.15.121:0/1535811303' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d", 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "", 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "" 2024-04-23T00:48:21.306 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:48:21.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:20.016068+0000 mgr.smithi121.otezso (mgr.14594) 7 : audit [DBG] from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:21.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:20.139576+0000 mgr.smithi121.otezso (mgr.14594) 8 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-04-23T00:48:21.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:20.394289+0000 mgr.smithi121.otezso (mgr.14594) 9 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:21.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:20.894086+0000 mon.smithi153 (mon.1) 43 : audit [DBG] from='client.? 172.21.15.121:0/1535811303' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:48:21.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 32669 32.25 MB/sec warmup 30 sec latency 15.283 ms 2024-04-23T00:48:21.803 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:48:22.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:21.307261+0000 mgr.smithi121.otezso (mgr.14594) 10 : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:22.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:21.804627+0000 mon.smithi121 (mon.0) 629 : audit [DBG] from='client.? 172.21.15.121:0/3485468679' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:48:22.602 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:21.307261+0000 mgr.smithi121.otezso (mgr.14594) 10 : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:22.602 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:21.804627+0000 mon.smithi121 (mon.0) 629 : audit [DBG] from='client.? 172.21.15.121:0/3485468679' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:48:22.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 33796 32.04 MB/sec warmup 31 sec latency 17.853 ms 2024-04-23T00:48:23.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 35131 32.37 MB/sec warmup 32 sec latency 29.987 ms 2024-04-23T00:48:24.125 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:22.140196+0000 mgr.smithi121.otezso (mgr.14594) 11 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-04-23T00:48:24.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:22.863305+0000 mon.smithi121 (mon.0) 630 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:24.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:23.229857+0000 mon.smithi121 (mon.0) 631 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:24.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:22.140196+0000 mgr.smithi121.otezso (mgr.14594) 11 : cluster [DBG] pgmap v7: 97 pgs: 97 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-04-23T00:48:24.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:22.863305+0000 mon.smithi121 (mon.0) 630 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:24.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:23.229857+0000 mon.smithi121 (mon.0) 631 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:24.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 36135 32.18 MB/sec warmup 33 sec latency 19.998 ms 2024-04-23T00:48:25.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:23.232890+0000 mgr.smithi121.otezso (mgr.14594) 12 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:48:25.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:23.232890+0000 mgr.smithi121.otezso (mgr.14594) 12 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:646b8c7d293d8ec31a40c69f7b3237b19c1be24d 2024-04-23T00:48:25.654 INFO:teuthology.orchestra.run.smithi121.stdout: 5 36994 31.79 MB/sec warmup 34 sec latency 21.892 ms 2024-04-23T00:48:26.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:24.140619+0000 mgr.smithi121.otezso (mgr.14594) 13 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 810 op/s 2024-04-23T00:48:26.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:24.140619+0000 mgr.smithi121.otezso (mgr.14594) 13 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 810 op/s 2024-04-23T00:48:26.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 37825 31.29 MB/sec warmup 35 sec latency 25.608 ms 2024-04-23T00:48:27.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 38799 31.50 MB/sec warmup 36 sec latency 20.364 ms 2024-04-23T00:48:28.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:26.141099+0000 mgr.smithi121.otezso (mgr.14594) 14 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 14 MiB/s wr, 1.31k op/s 2024-04-23T00:48:28.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:26.141099+0000 mgr.smithi121.otezso (mgr.14594) 14 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 14 MiB/s wr, 1.31k op/s 2024-04-23T00:48:28.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 39482 31.24 MB/sec warmup 37 sec latency 17.207 ms 2024-04-23T00:48:29.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:28.141483+0000 mgr.smithi121.otezso (mgr.14594) 15 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-04-23T00:48:29.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:28.141483+0000 mgr.smithi121.otezso (mgr.14594) 15 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 234 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-04-23T00:48:29.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 40018 30.60 MB/sec warmup 38 sec latency 27.546 ms 2024-04-23T00:48:30.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 41009 30.41 MB/sec warmup 39 sec latency 27.862 ms 2024-04-23T00:48:31.528 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:30.141881+0000 mgr.smithi121.otezso (mgr.14594) 16 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 236 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 15 MiB/s wr, 1.37k op/s 2024-04-23T00:48:31.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:30.141881+0000 mgr.smithi121.otezso (mgr.14594) 16 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 236 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 15 MiB/s wr, 1.37k op/s 2024-04-23T00:48:31.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 42020 30.35 MB/sec warmup 40 sec latency 38.150 ms 2024-04-23T00:48:32.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 42889 30.47 MB/sec warmup 41 sec latency 25.547 ms 2024-04-23T00:48:33.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:32.142316+0000 mgr.smithi121.otezso (mgr.14594) 17 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 17 MiB/s wr, 1.64k op/s 2024-04-23T00:48:33.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:32.142316+0000 mgr.smithi121.otezso (mgr.14594) 17 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 17 MiB/s wr, 1.64k op/s 2024-04-23T00:48:33.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 43750 30.13 MB/sec warmup 42 sec latency 20.729 ms 2024-04-23T00:48:34.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 44954 29.92 MB/sec warmup 43 sec latency 19.016 ms 2024-04-23T00:48:35.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:34.142715+0000 mgr.smithi121.otezso (mgr.14594) 18 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-23T00:48:35.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:34.142715+0000 mgr.smithi121.otezso (mgr.14594) 18 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 241 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-23T00:48:35.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 46092 30.31 MB/sec warmup 44 sec latency 16.584 ms 2024-04-23T00:48:36.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 46856 30.00 MB/sec warmup 45 sec latency 23.764 ms 2024-04-23T00:48:37.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:36.143345+0000 mgr.smithi121.otezso (mgr.14594) 19 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 254 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 17 MiB/s wr, 1.60k op/s 2024-04-23T00:48:37.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:36.143345+0000 mgr.smithi121.otezso (mgr.14594) 19 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 254 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 17 MiB/s wr, 1.60k op/s 2024-04-23T00:48:37.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 47899 29.76 MB/sec warmup 46 sec latency 19.437 ms 2024-04-23T00:48:38.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 49139 29.90 MB/sec warmup 47 sec latency 18.682 ms 2024-04-23T00:48:39.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:38.143728+0000 mgr.smithi121.otezso (mgr.14594) 20 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 254 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-04-23T00:48:39.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:38.143728+0000 mgr.smithi121.otezso (mgr.14594) 20 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 254 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-04-23T00:48:39.655 INFO:teuthology.orchestra.run.smithi121.stdout: 5 50214 30.07 MB/sec warmup 48 sec latency 27.439 ms 2024-04-23T00:48:40.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 51183 29.84 MB/sec warmup 49 sec latency 18.841 ms 2024-04-23T00:48:41.527 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:40.144204+0000 mgr.smithi121.otezso (mgr.14594) 21 : cluster [DBG] pgmap v16: 97 pgs: 97 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-04-23T00:48:41.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:40.144204+0000 mgr.smithi121.otezso (mgr.14594) 21 : cluster [DBG] pgmap v16: 97 pgs: 97 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-04-23T00:48:41.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 52247 29.75 MB/sec warmup 50 sec latency 18.963 ms 2024-04-23T00:48:42.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 53148 29.88 MB/sec warmup 51 sec latency 19.921 ms 2024-04-23T00:48:43.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:42.144635+0000 mgr.smithi121.otezso (mgr.14594) 22 : cluster [DBG] pgmap v17: 97 pgs: 97 active+clean; 277 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2024-04-23T00:48:43.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:42.144635+0000 mgr.smithi121.otezso (mgr.14594) 22 : cluster [DBG] pgmap v17: 97 pgs: 97 active+clean; 277 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.39k op/s 2024-04-23T00:48:43.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 53987 29.66 MB/sec warmup 52 sec latency 27.067 ms 2024-04-23T00:48:44.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 54884 29.43 MB/sec warmup 53 sec latency 22.138 ms 2024-04-23T00:48:45.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:44.144993+0000 mgr.smithi121.otezso (mgr.14594) 23 : cluster [DBG] pgmap v18: 97 pgs: 97 active+clean; 277 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-23T00:48:45.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:44.144993+0000 mgr.smithi121.otezso (mgr.14594) 23 : cluster [DBG] pgmap v18: 97 pgs: 97 active+clean; 277 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-23T00:48:45.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 55608 29.20 MB/sec warmup 54 sec latency 27.431 ms 2024-04-23T00:48:46.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 56611 29.39 MB/sec warmup 55 sec latency 25.506 ms 2024-04-23T00:48:47.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:46.145576+0000 mgr.smithi121.otezso (mgr.14594) 24 : cluster [DBG] pgmap v19: 97 pgs: 97 active+clean; 280 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 17 MiB/s wr, 1.65k op/s 2024-04-23T00:48:47.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:46.145576+0000 mgr.smithi121.otezso (mgr.14594) 24 : cluster [DBG] pgmap v19: 97 pgs: 97 active+clean; 280 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 17 MiB/s wr, 1.65k op/s 2024-04-23T00:48:47.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 57344 29.20 MB/sec warmup 56 sec latency 15.771 ms 2024-04-23T00:48:48.656 INFO:teuthology.orchestra.run.smithi121.stdout: 5 58075 28.98 MB/sec warmup 57 sec latency 20.155 ms 2024-04-23T00:48:49.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:48.146417+0000 mgr.smithi121.otezso (mgr.14594) 25 : cluster [DBG] pgmap v20: 97 pgs: 97 active+clean; 280 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-04-23T00:48:49.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:48.146417+0000 mgr.smithi121.otezso (mgr.14594) 25 : cluster [DBG] pgmap v20: 97 pgs: 97 active+clean; 280 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-04-23T00:48:49.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 58832 28.72 MB/sec warmup 58 sec latency 26.532 ms 2024-04-23T00:48:50.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 59498 28.57 MB/sec warmup 59 sec latency 31.398 ms 2024-04-23T00:48:51.486 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:50.146855+0000 mgr.smithi121.otezso (mgr.14594) 26 : cluster [DBG] pgmap v21: 97 pgs: 97 active+clean; 288 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.36k op/s 2024-04-23T00:48:51.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:50.146855+0000 mgr.smithi121.otezso (mgr.14594) 26 : cluster [DBG] pgmap v21: 97 pgs: 97 active+clean; 288 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.36k op/s 2024-04-23T00:48:51.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 60422 28.70 MB/sec warmup 60 sec latency 24.095 ms 2024-04-23T00:48:52.263 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:48:52.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 61190 28.45 MB/sec warmup 61 sec latency 15.751 ms 2024-04-23T00:48:53.088 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 running (5m) 30s ago 6m *:9093 *:9094 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (6m) 30s ago 6m - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (6m) 34s ago 6m - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 running (5m) 30s ago 6m *:3000 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (81s) 30s ago 80s - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (84s) 34s ago 83s - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 running (8m) 30s ago 8m *:9283 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:48:53.089 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 running (6m) 34s ago 6m *:8443 *:9283 16.2.0 24ecd6d5f14c 29976d00060b 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (8m) 30s ago 8m - 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (5m) 34s ago 5m - 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi121 smithi121 running (64s) 30s ago 64s *:2049 3.5 24ecd6d5f14c 6cbfa398c679 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi153 smithi153 running (69s) 34s ago 69s *:2049 3.5 24ecd6d5f14c 09a6f5208198 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 running (6m) 30s ago 6m *:9100 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 running (5m) 34s ago 5m *:9100 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (5m) 30s ago 5m - 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (4m) 30s ago 4m - 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (4m) 30s ago 4m - 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (4m) 30s ago 4m - 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (3m) 34s ago 3m - 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:48:53.090 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (3m) 34s ago 3m - 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:48:53.091 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (3m) 34s ago 3m - 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:48:53.091 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (2m) 34s ago 2m - 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:48:53.091 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 running (5m) 30s ago 6m *:9095 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:48:53.510 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:48:53.510 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:48:53.511 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 14 2024-04-23T00:48:53.512 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:48:53.512 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:48:53.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 61791 28.21 MB/sec warmup 62 sec latency 46.757 ms 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "0/23 ceph daemons upgraded", 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210 image on host smithi153" 2024-04-23T00:48:53.886 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:48:54.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:52.148716+0000 mgr.smithi121.otezso (mgr.14594) 27 : cluster [DBG] pgmap v22: 97 pgs: 97 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:52.246179+0000 mgr.smithi121.otezso (mgr.14594) 28 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:52.661585+0000 mgr.smithi121.otezso (mgr.14594) 29 : audit [DBG] from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:52.725444+0000 mon.smithi121 (mon.0) 632 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:52.725967+0000 mgr.smithi121.otezso (mgr.14594) 30 : cephadm [INF] Upgrade: Target is version 18.2.2-1237-g646b8c7d (unknown) 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:52.726154+0000 mgr.smithi121.otezso (mgr.14594) 31 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210, digests ['quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210'] 2024-04-23T00:48:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:52.727001+0000 mon.smithi153 (mon.1) 44 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:54.074 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:52.727928+0000 mgr.smithi121.otezso (mgr.14594) 32 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi121.otezso) 2024-04-23T00:48:54.074 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:53.511429+0000 mon.smithi121 (mon.0) 633 : audit [DBG] from='client.? 172.21.15.121:0/1236043841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:48:54.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:52.148716+0000 mgr.smithi121.otezso (mgr.14594) 27 : cluster [DBG] pgmap v22: 97 pgs: 97 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2024-04-23T00:48:54.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:52.246179+0000 mgr.smithi121.otezso (mgr.14594) 28 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:54.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:52.661585+0000 mgr.smithi121.otezso (mgr.14594) 29 : audit [DBG] from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:52.725444+0000 mon.smithi121 (mon.0) 632 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:52.725967+0000 mgr.smithi121.otezso (mgr.14594) 30 : cephadm [INF] Upgrade: Target is version 18.2.2-1237-g646b8c7d (unknown) 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:52.726154+0000 mgr.smithi121.otezso (mgr.14594) 31 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210, digests ['quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210'] 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:52.727001+0000 mon.smithi153 (mon.1) 44 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:52.727928+0000 mgr.smithi121.otezso (mgr.14594) 32 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi121.otezso) 2024-04-23T00:48:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:53.511429+0000 mon.smithi121 (mon.0) 633 : audit [DBG] from='client.? 172.21.15.121:0/1236043841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:48:54.363 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:48:54.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 62615 28.05 MB/sec warmup 63 sec latency 76.069 ms 2024-04-23T00:48:55.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:48:53.046840+0000 mgr.smithi121.otezso (mgr.14594) 33 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210 on smithi153 2024-04-23T00:48:55.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:53.077946+0000 mgr.smithi121.otezso (mgr.14594) 34 : audit [DBG] from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:55.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:53.886503+0000 mgr.smithi121.otezso (mgr.14594) 35 : audit [DBG] from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:55.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:48:54.363906+0000 mon.smithi153 (mon.1) 45 : audit [DBG] from='client.? 172.21.15.121:0/2320023162' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:48:55.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:48:53.046840+0000 mgr.smithi121.otezso (mgr.14594) 33 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210 on smithi153 2024-04-23T00:48:55.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:53.077946+0000 mgr.smithi121.otezso (mgr.14594) 34 : audit [DBG] from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:55.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:53.886503+0000 mgr.smithi121.otezso (mgr.14594) 35 : audit [DBG] from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:48:55.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:48:54.363906+0000 mon.smithi153 (mon.1) 45 : audit [DBG] from='client.? 172.21.15.121:0/2320023162' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:48:55.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 63677 28.25 MB/sec warmup 64 sec latency 17.328 ms 2024-04-23T00:48:56.071 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:54.149085+0000 mgr.smithi121.otezso (mgr.14594) 36 : cluster [DBG] pgmap v23: 97 pgs: 97 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 962 op/s 2024-04-23T00:48:56.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:54.149085+0000 mgr.smithi121.otezso (mgr.14594) 36 : cluster [DBG] pgmap v23: 97 pgs: 97 active+clean; 291 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 962 op/s 2024-04-23T00:48:56.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 64286 28.07 MB/sec warmup 65 sec latency 39.334 ms 2024-04-23T00:48:57.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 64768 27.75 MB/sec warmup 66 sec latency 35.766 ms 2024-04-23T00:48:58.074 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:56.149594+0000 mgr.smithi121.otezso (mgr.14594) 37 : cluster [DBG] pgmap v24: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 15 MiB/s wr, 1.44k op/s 2024-04-23T00:48:58.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:56.149594+0000 mgr.smithi121.otezso (mgr.14594) 37 : cluster [DBG] pgmap v24: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 15 MiB/s wr, 1.44k op/s 2024-04-23T00:48:58.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 65311 27.53 MB/sec warmup 67 sec latency 44.141 ms 2024-04-23T00:48:59.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:48:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:48:58.149925+0000 mgr.smithi121.otezso (mgr.14594) 38 : cluster [DBG] pgmap v25: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 913 op/s 2024-04-23T00:48:59.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:48:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:48:58.149925+0000 mgr.smithi121.otezso (mgr.14594) 38 : cluster [DBG] pgmap v25: 97 pgs: 97 active+clean; 308 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 913 op/s 2024-04-23T00:48:59.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 65889 27.30 MB/sec warmup 68 sec latency 121.337 ms 2024-04-23T00:49:00.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 66628 27.20 MB/sec warmup 69 sec latency 24.304 ms 2024-04-23T00:49:01.528 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:00.150395+0000 mgr.smithi121.otezso (mgr.14594) 39 : cluster [DBG] pgmap v26: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.27k op/s 2024-04-23T00:49:01.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:00.150395+0000 mgr.smithi121.otezso (mgr.14594) 39 : cluster [DBG] pgmap v26: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.27k op/s 2024-04-23T00:49:01.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 67365 27.25 MB/sec warmup 70 sec latency 25.871 ms 2024-04-23T00:49:02.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 67870 27.04 MB/sec warmup 71 sec latency 30.467 ms 2024-04-23T00:49:03.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:02.150737+0000 mgr.smithi121.otezso (mgr.14594) 40 : cluster [DBG] pgmap v27: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-04-23T00:49:03.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:02.150737+0000 mgr.smithi121.otezso (mgr.14594) 40 : cluster [DBG] pgmap v27: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-04-23T00:49:03.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 68222 26.75 MB/sec warmup 72 sec latency 46.040 ms 2024-04-23T00:49:04.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 69098 26.61 MB/sec warmup 73 sec latency 42.984 ms 2024-04-23T00:49:05.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:04.151104+0000 mgr.smithi121.otezso (mgr.14594) 41 : cluster [DBG] pgmap v28: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 862 op/s 2024-04-23T00:49:05.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:04.151104+0000 mgr.smithi121.otezso (mgr.14594) 41 : cluster [DBG] pgmap v28: 97 pgs: 97 active+clean; 313 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 862 op/s 2024-04-23T00:49:05.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 69967 26.56 MB/sec warmup 74 sec latency 21.471 ms 2024-04-23T00:49:06.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 70792 26.66 MB/sec warmup 75 sec latency 53.467 ms 2024-04-23T00:49:07.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:06.151784+0000 mgr.smithi121.otezso (mgr.14594) 42 : cluster [DBG] pgmap v29: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.26k op/s 2024-04-23T00:49:07.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:06.151784+0000 mgr.smithi121.otezso (mgr.14594) 42 : cluster [DBG] pgmap v29: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.26k op/s 2024-04-23T00:49:07.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 71283 26.51 MB/sec warmup 76 sec latency 32.091 ms 2024-04-23T00:49:08.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 71625 26.22 MB/sec warmup 77 sec latency 63.905 ms 2024-04-23T00:49:09.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:08.152112+0000 mgr.smithi121.otezso (mgr.14594) 43 : cluster [DBG] pgmap v30: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 790 op/s 2024-04-23T00:49:09.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:08.152112+0000 mgr.smithi121.otezso (mgr.14594) 43 : cluster [DBG] pgmap v30: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 790 op/s 2024-04-23T00:49:09.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 72010 25.98 MB/sec warmup 78 sec latency 54.040 ms 2024-04-23T00:49:10.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 72656 25.80 MB/sec warmup 79 sec latency 31.351 ms 2024-04-23T00:49:11.528 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:10.152545+0000 mgr.smithi121.otezso (mgr.14594) 44 : cluster [DBG] pgmap v31: 97 pgs: 97 active+clean; 306 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-23T00:49:11.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:10.152545+0000 mgr.smithi121.otezso (mgr.14594) 44 : cluster [DBG] pgmap v31: 97 pgs: 97 active+clean; 306 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-23T00:49:11.657 INFO:teuthology.orchestra.run.smithi121.stdout: 5 73206 25.63 MB/sec warmup 80 sec latency 29.837 ms 2024-04-23T00:49:12.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 74320 25.83 MB/sec warmup 81 sec latency 15.536 ms 2024-04-23T00:49:13.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:12.152954+0000 mgr.smithi121.otezso (mgr.14594) 45 : cluster [DBG] pgmap v32: 97 pgs: 97 active+clean; 308 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 774 op/s 2024-04-23T00:49:13.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:12.152954+0000 mgr.smithi121.otezso (mgr.14594) 45 : cluster [DBG] pgmap v32: 97 pgs: 97 active+clean; 308 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 774 op/s 2024-04-23T00:49:13.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 75193 25.79 MB/sec warmup 82 sec latency 16.178 ms 2024-04-23T00:49:14.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.145981+0000 mon.smithi153 (mon.1) 46 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-23T00:49:14.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146293+0000 mon.smithi153 (mon.1) 47 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-23T00:49:14.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146357+0000 mon.smithi121 (mon.0) 634 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-23T00:49:14.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146578+0000 mon.smithi153 (mon.1) 48 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146725+0000 mon.smithi121 (mon.0) 635 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-23T00:49:14.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146844+0000 mon.smithi153 (mon.1) 49 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.146958+0000 mon.smithi121 (mon.0) 636 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.147331+0000 mon.smithi121 (mon.0) 637 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.145981+0000 mon.smithi153 (mon.1) 46 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-23T00:49:14.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146293+0000 mon.smithi153 (mon.1) 47 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-23T00:49:14.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146357+0000 mon.smithi121 (mon.0) 634 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-23T00:49:14.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146578+0000 mon.smithi153 (mon.1) 48 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146725+0000 mon.smithi121 (mon.0) 635 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-23T00:49:14.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146844+0000 mon.smithi153 (mon.1) 49 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.146958+0000 mon.smithi121 (mon.0) 636 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.147331+0000 mon.smithi121 (mon.0) 637 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-23T00:49:14.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 75647 25.58 MB/sec warmup 83 sec latency 62.550 ms 2024-04-23T00:49:15.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:14.153359+0000 mgr.smithi121.otezso (mgr.14594) 46 : cluster [DBG] pgmap v33: 97 pgs: 97 active+clean; 308 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 736 op/s 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.218993+0000 mon.smithi121 (mon.0) 638 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.219091+0000 mon.smithi121 (mon.0) 639 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.219160+0000 mon.smithi121 (mon.0) 640 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.219258+0000 mon.smithi121 (mon.0) 641 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:14.219326+0000 mon.smithi121 (mon.0) 642 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.221552+0000 mon.smithi153 (mon.1) 50 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.222487+0000 mon.smithi121 (mon.0) 643 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.223830+0000 mon.smithi153 (mon.1) 51 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.224595+0000 mon.smithi121 (mon.0) 644 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.319236+0000 mon.smithi153 (mon.1) 52 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.319540+0000 mon.smithi121 (mon.0) 645 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.330622+0000 mon.smithi153 (mon.1) 53 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:14.330920+0000 mon.smithi121 (mon.0) 646 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:14.153359+0000 mgr.smithi121.otezso (mgr.14594) 46 : cluster [DBG] pgmap v33: 97 pgs: 97 active+clean; 308 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 736 op/s 2024-04-23T00:49:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.218993+0000 mon.smithi121 (mon.0) 638 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-23T00:49:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.219091+0000 mon.smithi121 (mon.0) 639 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-23T00:49:15.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.219160+0000 mon.smithi121 (mon.0) 640 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.219258+0000 mon.smithi121 (mon.0) 641 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:14.219326+0000 mon.smithi121 (mon.0) 642 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.221552+0000 mon.smithi153 (mon.1) 50 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.222487+0000 mon.smithi121 (mon.0) 643 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.223830+0000 mon.smithi153 (mon.1) 51 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.224595+0000 mon.smithi121 (mon.0) 644 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.319236+0000 mon.smithi153 (mon.1) 52 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.319540+0000 mon.smithi121 (mon.0) 645 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.330622+0000 mon.smithi153 (mon.1) 53 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:14.330920+0000 mon.smithi121 (mon.0) 646 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:15.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 76122 25.38 MB/sec warmup 84 sec latency 41.689 ms 2024-04-23T00:49:16.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:15.221894+0000 mon.smithi121 (mon.0) 647 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-04-23T00:49:16.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:15.221987+0000 mon.smithi121 (mon.0) 648 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-04-23T00:49:16.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:15.221894+0000 mon.smithi121 (mon.0) 647 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-04-23T00:49:16.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:15.221987+0000 mon.smithi121 (mon.0) 648 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-04-23T00:49:16.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 76577 25.20 MB/sec warmup 85 sec latency 51.208 ms 2024-04-23T00:49:17.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:16.153726+0000 mgr.smithi121.otezso (mgr.14594) 47 : cluster [DBG] pgmap v36: 97 pgs: 4 peering, 93 active+clean; 304 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-23T00:49:17.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:16.226744+0000 mon.smithi121 (mon.0) 649 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-04-23T00:49:17.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:16.233061+0000 mon.smithi121 (mon.0) 650 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-04-23T00:49:17.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:16.153726+0000 mgr.smithi121.otezso (mgr.14594) 47 : cluster [DBG] pgmap v36: 97 pgs: 4 peering, 93 active+clean; 304 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-23T00:49:17.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:16.226744+0000 mon.smithi121 (mon.0) 649 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-04-23T00:49:17.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:16.233061+0000 mon.smithi121 (mon.0) 650 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-04-23T00:49:17.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 77087 25.08 MB/sec warmup 86 sec latency 35.848 ms 2024-04-23T00:49:18.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:17.237784+0000 mon.smithi121 (mon.0) 651 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-04-23T00:49:18.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:17.237784+0000 mon.smithi121 (mon.0) 651 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-04-23T00:49:18.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 77566 24.98 MB/sec warmup 87 sec latency 33.301 ms 2024-04-23T00:49:19.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:18.154122+0000 mgr.smithi121.otezso (mgr.14594) 48 : cluster [DBG] pgmap v39: 97 pgs: 4 peering, 93 active+clean; 304 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.24k op/s 2024-04-23T00:49:19.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:18.243494+0000 mon.smithi121 (mon.0) 652 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-04-23T00:49:19.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:18.154122+0000 mgr.smithi121.otezso (mgr.14594) 48 : cluster [DBG] pgmap v39: 97 pgs: 4 peering, 93 active+clean; 304 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.24k op/s 2024-04-23T00:49:19.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:18.243494+0000 mon.smithi121 (mon.0) 652 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-04-23T00:49:19.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 77994 24.91 MB/sec warmup 88 sec latency 44.881 ms 2024-04-23T00:49:20.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:20.242421+0000 mon.smithi121 (mon.0) 653 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-04-23T00:49:20.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:20.242448+0000 mon.smithi121 (mon.0) 654 : cluster [INF] Cluster is now healthy 2024-04-23T00:49:20.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:20.242421+0000 mon.smithi121 (mon.0) 653 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-04-23T00:49:20.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:20.242448+0000 mon.smithi121 (mon.0) 654 : cluster [INF] Cluster is now healthy 2024-04-23T00:49:20.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 78550 24.80 MB/sec warmup 89 sec latency 33.119 ms 2024-04-23T00:49:21.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:20.154612+0000 mgr.smithi121.otezso (mgr.14594) 49 : cluster [DBG] pgmap v41: 97 pgs: 97 active+clean; 310 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 149 op/s; 3.2 MiB/s, 0 keys/s, 1 objects/s recovering 2024-04-23T00:49:21.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:20.154612+0000 mgr.smithi121.otezso (mgr.14594) 49 : cluster [DBG] pgmap v41: 97 pgs: 97 active+clean; 310 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 149 op/s; 3.2 MiB/s, 0 keys/s, 1 objects/s recovering 2024-04-23T00:49:21.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 79220 24.68 MB/sec warmup 90 sec latency 32.126 ms 2024-04-23T00:49:22.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 79869 24.54 MB/sec warmup 91 sec latency 49.775 ms 2024-04-23T00:49:23.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:22.155048+0000 mgr.smithi121.otezso (mgr.14594) 50 : cluster [DBG] pgmap v42: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.8 MiB/s wr, 505 op/s; 2.7 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:23.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:22.155048+0000 mgr.smithi121.otezso (mgr.14594) 50 : cluster [DBG] pgmap v42: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.8 MiB/s wr, 505 op/s; 2.7 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:23.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 80503 24.45 MB/sec warmup 92 sec latency 44.772 ms 2024-04-23T00:49:24.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 81391 24.55 MB/sec warmup 93 sec latency 73.670 ms 2024-04-23T00:49:24.786 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:49:25.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:24.155422+0000 mgr.smithi121.otezso (mgr.14594) 51 : cluster [DBG] pgmap v43: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 382 op/s; 2.0 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:25.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:24.771696+0000 mgr.smithi121.otezso (mgr.14594) 52 : audit [DBG] from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:25.597 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-04-23T00:49:25.597 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 running (5m) 62s ago 7m *:9093 *:9094 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (7m) 62s ago 7m - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (6m) 67s ago 6m - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 running (5m) 62s ago 7m *:3000 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (113s) 62s ago 113s - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (116s) 67s ago 116s - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 running (8m) 62s ago 8m *:9283 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 running (6m) 67s ago 6m *:8443 *:9283 16.2.0 24ecd6d5f14c 29976d00060b 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (8m) 62s ago 9m - 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (6m) 67s ago 6m - 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi121 smithi121 running (97s) 62s ago 96s *:2049 3.5 24ecd6d5f14c 6cbfa398c679 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi153 smithi153 running (102s) 67s ago 101s *:2049 3.5 24ecd6d5f14c 09a6f5208198 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 running (7m) 62s ago 7m *:9100 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:49:25.598 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 running (6m) 67s ago 6m *:9100 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (5m) 62s ago 5m - 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (5m) 62s ago 5m - 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (5m) 62s ago 5m - 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (4m) 62s ago 4m - 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (4m) 67s ago 4m - 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (4m) 67s ago 4m - 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (3m) 67s ago 3m - 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (3m) 67s ago 3m - 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:49:25.599 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 running (5m) 62s ago 6m *:9095 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:49:25.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:24.155422+0000 mgr.smithi121.otezso (mgr.14594) 51 : cluster [DBG] pgmap v43: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 382 op/s; 2.0 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:25.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:24.771696+0000 mgr.smithi121.otezso (mgr.14594) 52 : audit [DBG] from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:25.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 82238 24.52 MB/sec warmup 94 sec latency 30.775 ms 2024-04-23T00:49:26.078 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:49:26.078 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:49:26.078 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:49:26.078 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 14 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:49:26.079 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "0/23 ceph daemons upgraded", 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210 image on host smithi153" 2024-04-23T00:49:26.516 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:49:26.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:25.180918+0000 mgr.smithi121.otezso (mgr.14594) 53 : audit [DBG] from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:26.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:25.587752+0000 mgr.smithi121.otezso (mgr.14594) 54 : audit [DBG] from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:26.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:26.078165+0000 mon.smithi121 (mon.0) 655 : audit [DBG] from='client.? 172.21.15.121:0/2553264132' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:49:26.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:25.180918+0000 mgr.smithi121.otezso (mgr.14594) 53 : audit [DBG] from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:26.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:25.587752+0000 mgr.smithi121.otezso (mgr.14594) 54 : audit [DBG] from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:26.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:26.078165+0000 mon.smithi121 (mon.0) 655 : audit [DBG] from='client.? 172.21.15.121:0/2553264132' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:49:26.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 82932 24.43 MB/sec warmup 95 sec latency 56.040 ms 2024-04-23T00:49:27.022 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:49:27.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:26.156035+0000 mgr.smithi121.otezso (mgr.14594) 55 : cluster [DBG] pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.8 MiB/s wr, 919 op/s; 1.8 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:27.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:26.515543+0000 mgr.smithi121.otezso (mgr.14594) 56 : audit [DBG] from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:27.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:27.020378+0000 mon.smithi121 (mon.0) 656 : audit [DBG] from='client.? 172.21.15.121:0/672408427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:49:27.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:26.156035+0000 mgr.smithi121.otezso (mgr.14594) 55 : cluster [DBG] pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.8 MiB/s wr, 919 op/s; 1.8 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:27.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:26.515543+0000 mgr.smithi121.otezso (mgr.14594) 56 : audit [DBG] from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:27.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:27.020378+0000 mon.smithi121 (mon.0) 656 : audit [DBG] from='client.? 172.21.15.121:0/672408427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:49:27.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 83698 24.30 MB/sec warmup 96 sec latency 88.786 ms 2024-04-23T00:49:28.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 84419 24.29 MB/sec warmup 97 sec latency 27.734 ms 2024-04-23T00:49:29.571 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:28.156425+0000 mgr.smithi121.otezso (mgr.14594) 57 : cluster [DBG] pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.8 MiB/s wr, 819 op/s; 1.6 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:29.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:28.156425+0000 mgr.smithi121.otezso (mgr.14594) 57 : cluster [DBG] pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.8 MiB/s wr, 819 op/s; 1.6 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:29.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 84882 24.25 MB/sec warmup 98 sec latency 44.134 ms 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:29.317765+0000 mgr.smithi121.otezso (mgr.14594) 58 : cephadm [INF] Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.320007+0000 mon.smithi121 (mon.0) 657 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.324078+0000 mon.smithi121 (mon.0) 658 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.334970+0000 mon.smithi153 (mon.1) 54 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.335308+0000 mon.smithi121 (mon.0) 659 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.336250+0000 mon.smithi153 (mon.1) 55 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:29.337062+0000 mon.smithi153 (mon.1) 56 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:49:30.321 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:29.337753+0000 mgr.smithi121.otezso (mgr.14594) 59 : cephadm [INF] Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T00:49:30.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:29.317765+0000 mgr.smithi121.otezso (mgr.14594) 58 : cephadm [INF] Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.320007+0000 mon.smithi121 (mon.0) 657 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.324078+0000 mon.smithi121 (mon.0) 658 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.334970+0000 mon.smithi153 (mon.1) 54 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.335308+0000 mon.smithi121 (mon.0) 659 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.336250+0000 mon.smithi153 (mon.1) 55 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:29.337062+0000 mon.smithi153 (mon.1) 56 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:49:30.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:29.337753+0000 mgr.smithi121.otezso (mgr.14594) 59 : cephadm [INF] Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T00:49:30.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 85258 24.15 MB/sec warmup 99 sec latency 34.099 ms 2024-04-23T00:49:31.562 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:30.156951+0000 mgr.smithi121.otezso (mgr.14594) 60 : cluster [DBG] pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 981 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:31.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:30.156951+0000 mgr.smithi121.otezso (mgr.14594) 60 : cluster [DBG] pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 981 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-04-23T00:49:31.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 85761 24.00 MB/sec warmup 100 sec latency 37.258 ms 2024-04-23T00:49:32.659 INFO:teuthology.orchestra.run.smithi121.stdout: 5 86238 23.86 MB/sec warmup 101 sec latency 38.918 ms 2024-04-23T00:49:33.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:32.157288+0000 mgr.smithi121.otezso (mgr.14594) 61 : cluster [DBG] pgmap v47: 97 pgs: 97 active+clean; 315 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 1.00k op/s 2024-04-23T00:49:33.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:32.932920+0000 mon.smithi121 (mon.0) 660 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:33.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:32.934499+0000 mon.smithi153 (mon.1) 57 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:33.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:32.157288+0000 mgr.smithi121.otezso (mgr.14594) 61 : cluster [DBG] pgmap v47: 97 pgs: 97 active+clean; 315 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 1.00k op/s 2024-04-23T00:49:33.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:32.932920+0000 mon.smithi121 (mon.0) 660 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:33.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:32.934499+0000 mon.smithi153 (mon.1) 57 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:33.658 INFO:teuthology.orchestra.run.smithi121.stdout: 5 86898 23.73 MB/sec warmup 102 sec latency 57.727 ms 2024-04-23T00:49:34.659 INFO:teuthology.orchestra.run.smithi121.stdout: 5 87505 23.66 MB/sec warmup 103 sec latency 33.544 ms 2024-04-23T00:49:35.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:34.157641+0000 mgr.smithi121.otezso (mgr.14594) 62 : cluster [DBG] pgmap v48: 97 pgs: 97 active+clean; 315 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-04-23T00:49:35.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:34.157641+0000 mgr.smithi121.otezso (mgr.14594) 62 : cluster [DBG] pgmap v48: 97 pgs: 97 active+clean; 315 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-04-23T00:49:35.659 INFO:teuthology.orchestra.run.smithi121.stdout: 5 88524 23.79 MB/sec warmup 104 sec latency 22.680 ms 2024-04-23T00:49:36.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 89342 23.77 MB/sec warmup 105 sec latency 25.808 ms 2024-04-23T00:49:37.507 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:36.158252+0000 mgr.smithi121.otezso (mgr.14594) 63 : cluster [DBG] pgmap v49: 97 pgs: 97 active+clean; 315 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.17k op/s 2024-04-23T00:49:37.507 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:36.943177+0000 mon.smithi121 (mon.0) 661 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:37.508 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:37.332189+0000 mon.smithi121 (mon.0) 662 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:37.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:36.158252+0000 mgr.smithi121.otezso (mgr.14594) 63 : cluster [DBG] pgmap v49: 97 pgs: 97 active+clean; 315 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.17k op/s 2024-04-23T00:49:37.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:36.943177+0000 mon.smithi121 (mon.0) 661 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:37.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:37.332189+0000 mon.smithi121 (mon.0) 662 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' 2024-04-23T00:49:37.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 90010 23.70 MB/sec warmup 106 sec latency 27.994 ms 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:37.338288+0000 mon.smithi153 (mon.1) 58 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:37.341903+0000 mgr.smithi121.otezso (mgr.14594) 64 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi121.otezso) 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:37.344991+0000 mgr.smithi121.otezso (mgr.14594) 65 : cephadm [INF] Failing over to other MGR 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:37.345618+0000 mon.smithi153 (mon.1) 59 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr fail", "who": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:37.345903+0000 mon.smithi121 (mon.0) 663 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr fail", "who": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:38.387 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:37.349912+0000 mon.smithi121 (mon.0) 664 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-04-23T00:49:38.388 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:38.191148+0000 mon.smithi121 (mon.0) 665 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:37.338288+0000 mon.smithi153 (mon.1) 58 : audit [DBG] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:37.341903+0000 mgr.smithi121.otezso (mgr.14594) 64 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi121.otezso) 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:37.344991+0000 mgr.smithi121.otezso (mgr.14594) 65 : cephadm [INF] Failing over to other MGR 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:37.345618+0000 mon.smithi153 (mon.1) 59 : audit [INF] from='mgr.14594 172.21.15.121:0/493470951' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr fail", "who": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:37.345903+0000 mon.smithi121 (mon.0) 663 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr fail", "who": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:37.349912+0000 mon.smithi121 (mon.0) 664 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-04-23T00:49:38.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:38.191148+0000 mon.smithi121 (mon.0) 665 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T00:49:38.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 90788 23.59 MB/sec warmup 107 sec latency 33.217 ms 2024-04-23T00:49:39.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:38.350307+0000 mon.smithi121 (mon.0) 666 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "mgr fail", "who": "smithi121.otezso"}]': finished 2024-04-23T00:49:39.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:38.350385+0000 mon.smithi121 (mon.0) 667 : cluster [DBG] mgrmap e24: smithi153.olhmgd(active, starting, since 1.00379s), standbys: smithi121.otezso 2024-04-23T00:49:39.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:38.350307+0000 mon.smithi121 (mon.0) 666 : audit [INF] from='mgr.14594 ' entity='mgr.smithi121.otezso' cmd='[{"prefix": "mgr fail", "who": "smithi121.otezso"}]': finished 2024-04-23T00:49:39.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:38.350385+0000 mon.smithi121 (mon.0) 667 : cluster [DBG] mgrmap e24: smithi153.olhmgd(active, starting, since 1.00379s), standbys: smithi121.otezso 2024-04-23T00:49:39.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 91547 23.60 MB/sec warmup 108 sec latency 31.063 ms 2024-04-23T00:49:40.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 92391 23.69 MB/sec warmup 109 sec latency 30.903 ms 2024-04-23T00:49:41.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 93008 23.58 MB/sec warmup 110 sec latency 24.313 ms 2024-04-23T00:49:42.505 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:42.152303+0000 mon.smithi121 (mon.0) 668 : cluster [INF] Active manager daemon smithi153.olhmgd restarted 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:42.152955+0000 mon.smithi121 (mon.0) 669 : cluster [INF] Activating manager daemon smithi153.olhmgd 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.158997+0000 mon.smithi153 (mon.1) 60 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:42.159121+0000 mon.smithi121 (mon.0) 670 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.159663+0000 mon.smithi153 (mon.1) 61 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.160673+0000 mon.smithi153 (mon.1) 62 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:49:42.506 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.160932+0000 mon.smithi153 (mon.1) 63 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:42.152303+0000 mon.smithi121 (mon.0) 668 : cluster [INF] Active manager daemon smithi153.olhmgd restarted 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:42.152955+0000 mon.smithi121 (mon.0) 669 : cluster [INF] Activating manager daemon smithi153.olhmgd 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.158997+0000 mon.smithi153 (mon.1) 60 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:42.159121+0000 mon.smithi121 (mon.0) 670 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.159663+0000 mon.smithi153 (mon.1) 61 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.160673+0000 mon.smithi153 (mon.1) 62 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T00:49:42.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.160932+0000 mon.smithi153 (mon.1) 63 : audit [DBG] from='mgr.? 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:49:42.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 93500 23.49 MB/sec warmup 111 sec latency 44.475 ms 2024-04-23T00:49:43.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:42.211584+0000 mon.smithi121 (mon.0) 671 : cluster [DBG] mgrmap e25: smithi153.olhmgd(active, starting, since 0.0588312s), standbys: smithi121.otezso 2024-04-23T00:49:43.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.220432+0000 mon.smithi153 (mon.1) 64 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.220582+0000 mon.smithi153 (mon.1) 65 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.221306+0000 mon.smithi153 (mon.1) 66 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.221513+0000 mon.smithi153 (mon.1) 67 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.221845+0000 mon.smithi153 (mon.1) 68 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.222015+0000 mon.smithi153 (mon.1) 69 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.222184+0000 mon.smithi153 (mon.1) 70 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.222394+0000 mon.smithi153 (mon.1) 71 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.222611+0000 mon.smithi153 (mon.1) 72 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.222810+0000 mon.smithi153 (mon.1) 73 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.223049+0000 mon.smithi153 (mon.1) 74 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.223339+0000 mon.smithi153 (mon.1) 75 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.223638+0000 mon.smithi153 (mon.1) 76 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.223934+0000 mon.smithi153 (mon.1) 77 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:49:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.224794+0000 mon.smithi153 (mon.1) 78 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.225031+0000 mon.smithi153 (mon.1) 79 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.225577+0000 mon.smithi153 (mon.1) 80 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:42.333579+0000 mon.smithi121 (mon.0) 672 : cluster [INF] Manager daemon smithi153.olhmgd is now available 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:42.345862+0000 mgr.smithi153.olhmgd (mgr.24421) 1 : cephadm [INF] inventory: adjusted host smithi121 addr 'smithi121' -> '172.21.15.121' 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.350494+0000 mon.smithi121 (mon.0) 673 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.361721+0000 mon.smithi121 (mon.0) 674 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:42.362110+0000 mgr.smithi153.olhmgd (mgr.24421) 2 : cephadm [INF] Queued nfs.foo for migration 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.367593+0000 mon.smithi153 (mon.1) 81 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.384383+0000 mon.smithi121 (mon.0) 675 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.394633+0000 mon.smithi153 (mon.1) 82 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.420506+0000 mon.smithi153 (mon.1) 83 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.421495+0000 mon.smithi121 (mon.0) 676 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:43.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.430240+0000 mon.smithi121 (mon.0) 677 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.433547+0000 mon.smithi153 (mon.1) 84 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:43.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.434083+0000 mon.smithi121 (mon.0) 678 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:43.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.814348+0000 mon.smithi121 (mon.0) 679 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:42.821401+0000 mon.smithi121 (mon.0) 680 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:42.211584+0000 mon.smithi121 (mon.0) 671 : cluster [DBG] mgrmap e25: smithi153.olhmgd(active, starting, since 0.0588312s), standbys: smithi121.otezso 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.220432+0000 mon.smithi153 (mon.1) 64 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.220582+0000 mon.smithi153 (mon.1) 65 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.221306+0000 mon.smithi153 (mon.1) 66 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.221513+0000 mon.smithi153 (mon.1) 67 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.221845+0000 mon.smithi153 (mon.1) 68 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.222015+0000 mon.smithi153 (mon.1) 69 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.222184+0000 mon.smithi153 (mon.1) 70 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T00:49:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.222394+0000 mon.smithi153 (mon.1) 71 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.222611+0000 mon.smithi153 (mon.1) 72 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.222810+0000 mon.smithi153 (mon.1) 73 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.223049+0000 mon.smithi153 (mon.1) 74 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.223339+0000 mon.smithi153 (mon.1) 75 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.223638+0000 mon.smithi153 (mon.1) 76 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.223934+0000 mon.smithi153 (mon.1) 77 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.224794+0000 mon.smithi153 (mon.1) 78 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.225031+0000 mon.smithi153 (mon.1) 79 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.225577+0000 mon.smithi153 (mon.1) 80 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:42.333579+0000 mon.smithi121 (mon.0) 672 : cluster [INF] Manager daemon smithi153.olhmgd is now available 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:42.345862+0000 mgr.smithi153.olhmgd (mgr.24421) 1 : cephadm [INF] inventory: adjusted host smithi121 addr 'smithi121' -> '172.21.15.121' 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.350494+0000 mon.smithi121 (mon.0) 673 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.361721+0000 mon.smithi121 (mon.0) 674 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:42.362110+0000 mgr.smithi153.olhmgd (mgr.24421) 2 : cephadm [INF] Queued nfs.foo for migration 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.367593+0000 mon.smithi153 (mon.1) 81 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.384383+0000 mon.smithi121 (mon.0) 675 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.394633+0000 mon.smithi153 (mon.1) 82 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.420506+0000 mon.smithi153 (mon.1) 83 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.421495+0000 mon.smithi121 (mon.0) 676 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/mirror_snapshot_schedule"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.430240+0000 mon.smithi121 (mon.0) 677 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.433547+0000 mon.smithi153 (mon.1) 84 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.434083+0000 mon.smithi121 (mon.0) 678 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi153.olhmgd/trash_purge_schedule"}]: dispatch 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.814348+0000 mon.smithi121 (mon.0) 679 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:42.821401+0000 mon.smithi121 (mon.0) 680 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:43.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 94053 23.34 MB/sec warmup 112 sec latency 28.387 ms 2024-04-23T00:49:44.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:42.840747+0000 mgr.smithi153.olhmgd (mgr.24421) 3 : cephadm [INF] Deploying cephadm binary to smithi153 2024-04-23T00:49:44.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:42.848698+0000 mgr.smithi153.olhmgd (mgr.24421) 4 : cephadm [INF] Deploying cephadm binary to smithi121 2024-04-23T00:49:44.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:43.231911+0000 mon.smithi121 (mon.0) 681 : cluster [DBG] mgrmap e26: smithi153.olhmgd(active, since 1.07915s), standbys: smithi121.otezso 2024-04-23T00:49:44.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:43.245348+0000 mgr.smithi153.olhmgd (mgr.24421) 5 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:44.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:43.705203+0000 mgr.smithi153.olhmgd (mgr.24421) 6 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Bus STARTING 2024-04-23T00:49:44.626 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:42.840747+0000 mgr.smithi153.olhmgd (mgr.24421) 3 : cephadm [INF] Deploying cephadm binary to smithi153 2024-04-23T00:49:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:42.848698+0000 mgr.smithi153.olhmgd (mgr.24421) 4 : cephadm [INF] Deploying cephadm binary to smithi121 2024-04-23T00:49:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:43.231911+0000 mon.smithi121 (mon.0) 681 : cluster [DBG] mgrmap e26: smithi153.olhmgd(active, since 1.07915s), standbys: smithi121.otezso 2024-04-23T00:49:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:43.245348+0000 mgr.smithi153.olhmgd (mgr.24421) 5 : cluster [DBG] pgmap v3: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:43.705203+0000 mgr.smithi153.olhmgd (mgr.24421) 6 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Bus STARTING 2024-04-23T00:49:44.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 94465 23.23 MB/sec warmup 113 sec latency 34.062 ms 2024-04-23T00:49:45.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:43.833840+0000 mgr.smithi153.olhmgd (mgr.24421) 7 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Serving on http://172.21.15.153:8765 2024-04-23T00:49:45.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:43.989024+0000 mgr.smithi153.olhmgd (mgr.24421) 8 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Serving on https://172.21.15.153:7150 2024-04-23T00:49:45.379 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:49:43.989129+0000 mgr.smithi153.olhmgd (mgr.24421) 9 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Bus STARTED 2024-04-23T00:49:45.379 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:44.224965+0000 mgr.smithi153.olhmgd (mgr.24421) 10 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:45.379 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:44.239011+0000 mon.smithi121 (mon.0) 682 : cluster [DBG] mgrmap e27: smithi153.olhmgd(active, since 2s), standbys: smithi121.otezso 2024-04-23T00:49:45.379 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:44.386539+0000 mon.smithi153 (mon.1) 85 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:45.379 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:44.386887+0000 mon.smithi121 (mon.0) 683 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:45.020604+0000 mon.smithi121 (mon.0) 684 : cluster [DBG] Standby manager daemon smithi121.otezso restarted 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:45.020709+0000 mon.smithi121 (mon.0) 685 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.021902+0000 mon.smithi153 (mon.1) 86 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/crt"}]: dispatch 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.025522+0000 mon.smithi153 (mon.1) 87 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.027478+0000 mon.smithi153 (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/key"}]: dispatch 2024-04-23T00:49:45.380 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.027824+0000 mon.smithi153 (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:49:45.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:43.833840+0000 mgr.smithi153.olhmgd (mgr.24421) 7 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Serving on http://172.21.15.153:8765 2024-04-23T00:49:45.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:43.989024+0000 mgr.smithi153.olhmgd (mgr.24421) 8 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Serving on https://172.21.15.153:7150 2024-04-23T00:49:45.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:49:43.989129+0000 mgr.smithi153.olhmgd (mgr.24421) 9 : cephadm [INF] [23/Apr/2024:00:49:43] ENGINE Bus STARTED 2024-04-23T00:49:45.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:44.224965+0000 mgr.smithi153.olhmgd (mgr.24421) 10 : cluster [DBG] pgmap v4: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:44.239011+0000 mon.smithi121 (mon.0) 682 : cluster [DBG] mgrmap e27: smithi153.olhmgd(active, since 2s), standbys: smithi121.otezso 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:44.386539+0000 mon.smithi153 (mon.1) 85 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:44.386887+0000 mon.smithi121 (mon.0) 683 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:45.020604+0000 mon.smithi121 (mon.0) 684 : cluster [DBG] Standby manager daemon smithi121.otezso restarted 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:45.020709+0000 mon.smithi121 (mon.0) 685 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.021902+0000 mon.smithi153 (mon.1) 86 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/crt"}]: dispatch 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.025522+0000 mon.smithi153 (mon.1) 87 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.027478+0000 mon.smithi153 (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/key"}]: dispatch 2024-04-23T00:49:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.027824+0000 mon.smithi153 (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.121:0/1834537093' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T00:49:45.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 94909 23.16 MB/sec warmup 114 sec latency 41.572 ms 2024-04-23T00:49:46.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.245719+0000 mon.smithi121 (mon.0) 686 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:49:46.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:45.245791+0000 mon.smithi121 (mon.0) 687 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-04-23T00:49:46.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:45.246306+0000 mon.smithi121 (mon.0) 688 : cluster [DBG] mgrmap e28: smithi153.olhmgd(active, since 3s), standbys: smithi121.otezso 2024-04-23T00:49:46.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.255615+0000 mon.smithi153 (mon.1) 90 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:46.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:45.256002+0000 mon.smithi121 (mon.0) 689 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.245719+0000 mon.smithi121 (mon.0) 686 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:49:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:45.245791+0000 mon.smithi121 (mon.0) 687 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-04-23T00:49:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:45.246306+0000 mon.smithi121 (mon.0) 688 : cluster [DBG] mgrmap e28: smithi153.olhmgd(active, since 3s), standbys: smithi121.otezso 2024-04-23T00:49:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.255615+0000 mon.smithi153 (mon.1) 90 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:45.256002+0000 mon.smithi121 (mon.0) 689 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:49:46.660 INFO:teuthology.orchestra.run.smithi121.stdout: 5 95582 23.17 MB/sec warmup 115 sec latency 27.175 ms 2024-04-23T00:49:47.249 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:46.225412+0000 mgr.smithi153.olhmgd (mgr.24421) 11 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:47.249 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:46.250511+0000 mon.smithi121 (mon.0) 690 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:49:47.250 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:46.250595+0000 mon.smithi121 (mon.0) 691 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-04-23T00:49:47.250 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:47.058988+0000 mon.smithi121 (mon.0) 692 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:47.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:46.225412+0000 mgr.smithi153.olhmgd (mgr.24421) 11 : cluster [DBG] pgmap v6: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:47.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:46.250511+0000 mon.smithi121 (mon.0) 690 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:49:47.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:46.250595+0000 mon.smithi121 (mon.0) 691 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-04-23T00:49:47.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:47.058988+0000 mon.smithi121 (mon.0) 692 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:47.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 96151 23.12 MB/sec warmup 116 sec latency 34.626 ms 2024-04-23T00:49:48.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 96621 23.00 MB/sec warmup 117 sec latency 37.414 ms 2024-04-23T00:49:49.500 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:48.225722+0000 mgr.smithi153.olhmgd (mgr.24421) 12 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:49.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:48.225722+0000 mgr.smithi153.olhmgd (mgr.24421) 12 : cluster [DBG] pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-23T00:49:49.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 97235 22.92 MB/sec warmup 118 sec latency 38.863 ms 2024-04-23T00:49:50.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:49.155916+0000 mon.smithi121 (mon.0) 693 : cluster [DBG] mgrmap e29: smithi153.olhmgd(active, since 7s), standbys: smithi121.otezso 2024-04-23T00:49:50.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:49.691698+0000 mon.smithi121 (mon.0) 694 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:50.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:49.155916+0000 mon.smithi121 (mon.0) 693 : cluster [DBG] mgrmap e29: smithi153.olhmgd(active, since 7s), standbys: smithi121.otezso 2024-04-23T00:49:50.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:49.691698+0000 mon.smithi121 (mon.0) 694 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:50.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 98095 22.87 MB/sec warmup 119 sec latency 26.626 ms 2024-04-23T00:49:51.529 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:50.226399+0000 mgr.smithi153.olhmgd (mgr.24421) 13 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-04-23T00:49:51.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:50.226399+0000 mgr.smithi153.olhmgd (mgr.24421) 13 : cluster [DBG] pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-04-23T00:49:52.661 INFO:teuthology.orchestra.run.smithi121.stdout: 5 99152 15.08 MB/sec execute 1 sec latency 469.487 ms 2024-04-23T00:49:53.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:51.629046+0000 mon.smithi121 (mon.0) 695 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-23T00:49:53.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:51.629296+0000 mon.smithi153 (mon.1) 91 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:53.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:51.629491+0000 mon.smithi153 (mon.1) 92 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:53.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.133578+0000 mon.smithi121 (mon.0) 696 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-23T00:49:53.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:51.629046+0000 mon.smithi121 (mon.0) 695 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-23T00:49:53.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:51.629296+0000 mon.smithi153 (mon.1) 91 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:53.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:51.629491+0000 mon.smithi153 (mon.1) 92 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:53.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.133578+0000 mon.smithi121 (mon.0) 696 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-23T00:49:53.662 INFO:teuthology.orchestra.run.smithi121.stdout: 5 99600 15.15 MB/sec execute 2 sec latency 32.051 ms 2024-04-23T00:49:54.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.135385+0000 mon.smithi153 (mon.1) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-23T00:49:54.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:52.226764+0000 mgr.smithi153.olhmgd (mgr.24421) 14 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-04-23T00:49:54.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.718186+0000 mon.smithi153 (mon.1) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-23T00:49:54.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.719433+0000 mon.smithi153 (mon.1) 95 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.722918+0000 mon.smithi153 (mon.1) 96 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.727868+0000 mon.smithi121 (mon.0) 697 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:52.751966+0000 mon.smithi121 (mon.0) 698 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:53.546339+0000 mon.smithi121 (mon.0) 699 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:53.554831+0000 mon.smithi121 (mon.0) 700 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.135385+0000 mon.smithi153 (mon.1) 93 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:52.226764+0000 mgr.smithi153.olhmgd (mgr.24421) 14 : cluster [DBG] pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.718186+0000 mon.smithi153 (mon.1) 94 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.719433+0000 mon.smithi153 (mon.1) 95 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.722918+0000 mon.smithi153 (mon.1) 96 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T00:49:54.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.727868+0000 mon.smithi121 (mon.0) 697 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:52.751966+0000 mon.smithi121 (mon.0) 698 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:53.546339+0000 mon.smithi121 (mon.0) 699 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:53.554831+0000 mon.smithi121 (mon.0) 700 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:54.662 INFO:teuthology.orchestra.run.smithi121.stdout: 5 100042 12.48 MB/sec execute 3 sec latency 33.237 ms 2024-04-23T00:49:55.662 INFO:teuthology.orchestra.run.smithi121.stdout: 5 100335 10.64 MB/sec execute 4 sec latency 39.021 ms 2024-04-23T00:49:56.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:54.227270+0000 mgr.smithi153.olhmgd (mgr.24421) 15 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 9.8 MiB/s wr, 918 op/s 2024-04-23T00:49:56.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:54.724088+0000 mon.smithi121 (mon.0) 701 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:54.733770+0000 mon.smithi121 (mon.0) 702 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:55.571410+0000 mon.smithi121 (mon.0) 703 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:55.577586+0000 mon.smithi121 (mon.0) 704 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:54.227270+0000 mgr.smithi153.olhmgd (mgr.24421) 15 : cluster [DBG] pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 9.8 MiB/s wr, 918 op/s 2024-04-23T00:49:56.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:54.724088+0000 mon.smithi121 (mon.0) 701 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:54.733770+0000 mon.smithi121 (mon.0) 702 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:55.571410+0000 mon.smithi121 (mon.0) 703 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:55.577586+0000 mon.smithi121 (mon.0) 704 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:56.662 INFO:teuthology.orchestra.run.smithi121.stdout: 5 100974 11.62 MB/sec execute 5 sec latency 34.761 ms 2024-04-23T00:49:57.457 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:49:57.662 INFO:teuthology.orchestra.run.smithi121.stdout: 5 101502 10.99 MB/sec execute 6 sec latency 50.329 ms 2024-04-23T00:49:57.727 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:56.227602+0000 mgr.smithi153.olhmgd (mgr.24421) 16 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 8.8 MiB/s wr, 824 op/s 2024-04-23T00:49:57.727 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:57.394489+0000 mon.smithi153 (mon.1) 97 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:49:57.727 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:57.490800+0000 mon.smithi121 (mon.0) 705 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:57.727 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:57.498402+0000 mon.smithi121 (mon.0) 706 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:57.999 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:56.227602+0000 mgr.smithi153.olhmgd (mgr.24421) 16 : cluster [DBG] pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 8.8 MiB/s wr, 824 op/s 2024-04-23T00:49:57.999 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:57.394489+0000 mon.smithi153 (mon.1) 97 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:49:57.999 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:57.490800+0000 mon.smithi121 (mon.0) 705 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:57.999 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:57.498402+0000 mon.smithi121 (mon.0) 706 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:49:58.247 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:49:58.247 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (6m) 5s ago 8m 14.9M - 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:49:58.247 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (8m) 5s ago 8m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:49:58.247 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (7m) 3s ago 7m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:49:58.247 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (6m) 5s ago 7m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (2m) 5s ago 2m 12.2M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (2m) 3s ago 2m 38.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (9m) 5s ago 9m 358M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (25s) 3s ago 7m 482M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (9m) 5s ago 9m 58.7M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (7m) 3s ago 7m 55.1M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi121 smithi121 *:2049 running (2m) 5s ago 2m 244M - 3.5 24ecd6d5f14c 6cbfa398c679 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.smithi153 smithi153 *:2049 running (2m) 3s ago 2m 81.4M - 3.5 24ecd6d5f14c 09a6f5208198 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (7m) 5s ago 7m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (6m) 3s ago 6m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (6m) 5s ago 6m 373M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (6m) 5s ago 6m 291M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (5m) 5s ago 5m 235M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (5m) 5s ago 5m 336M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:49:58.248 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (4m) 3s ago 4m 361M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:49:58.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (4m) 3s ago 4m 377M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:49:58.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (4m) 3s ago 4m 319M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:49:58.249 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (3m) 3s ago 3m 463M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:49:58.249 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (6m) 5s ago 7m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:49:58.663 INFO:teuthology.orchestra.run.smithi121.stdout: 5 102168 12.67 MB/sec execute 7 sec latency 59.010 ms 2024-04-23T00:49:58.831 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:49:58.831 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:49:58.831 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:49:58.831 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:58.831 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:49:58.832 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:49:59.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:57.441192+0000 mgr.smithi153.olhmgd (mgr.24421) 17 : audit [DBG] from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:59.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:57.441192+0000 mgr.smithi153.olhmgd (mgr.24421) 17 : audit [DBG] from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:59.311 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:49:59.311 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:49:59.311 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:49:59.312 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:49:59.663 INFO:teuthology.orchestra.run.smithi121.stdout: 5 102780 14.19 MB/sec execute 8 sec latency 23.778 ms 2024-04-23T00:49:59.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:57.846816+0000 mgr.smithi153.olhmgd (mgr.24421) 18 : audit [DBG] from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:59.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:49:58.227969+0000 mgr.smithi153.olhmgd (mgr.24421) 19 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 7.3 MiB/s wr, 688 op/s 2024-04-23T00:49:59.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:58.233958+0000 mgr.smithi153.olhmgd (mgr.24421) 20 : audit [DBG] from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:49:59.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:49:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:58.828401+0000 mon.smithi121 (mon.0) 707 : audit [DBG] from='client.? 172.21.15.121:0/654182823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:49:59.877 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:50:00.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:57.846816+0000 mgr.smithi153.olhmgd (mgr.24421) 18 : audit [DBG] from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:00.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:49:58.227969+0000 mgr.smithi153.olhmgd (mgr.24421) 19 : cluster [DBG] pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 7.3 MiB/s wr, 688 op/s 2024-04-23T00:50:00.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:58.233958+0000 mgr.smithi153.olhmgd (mgr.24421) 20 : audit [DBG] from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:00.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:49:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:58.828401+0000 mon.smithi121 (mon.0) 707 : audit [DBG] from='client.? 172.21.15.121:0/654182823' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:50:00.663 INFO:teuthology.orchestra.run.smithi121.stdout: 5 103295 14.09 MB/sec execute 9 sec latency 22.609 ms 2024-04-23T00:50:00.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:59.311229+0000 mgr.smithi153.olhmgd (mgr.24421) 21 : audit [DBG] from='client.24465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:00.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:49:59.875493+0000 mon.smithi121 (mon.0) 708 : audit [DBG] from='client.? 172.21.15.121:0/3018668748' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:50:00.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:00.000094+0000 mon.smithi121 (mon.0) 709 : cluster [INF] overall HEALTH_OK 2024-04-23T00:50:01.126 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:59.311229+0000 mgr.smithi153.olhmgd (mgr.24421) 21 : audit [DBG] from='client.24465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:01.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:49:59.875493+0000 mon.smithi121 (mon.0) 708 : audit [DBG] from='client.? 172.21.15.121:0/3018668748' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:50:01.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:00.000094+0000 mon.smithi121 (mon.0) 709 : cluster [INF] overall HEALTH_OK 2024-04-23T00:50:01.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 103650 13.43 MB/sec execute 10 sec latency 218.125 ms 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:00.228462+0000 mgr.smithi153.olhmgd (mgr.24421) 22 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 996 op/s 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:01.407805+0000 mon.smithi121 (mon.0) 710 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:01.413554+0000 mon.smithi121 (mon.0) 711 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:01.419205+0000 mon.smithi121 (mon.0) 712 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:01.440755+0000 mon.smithi153 (mon.1) 98 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:50:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:01.441142+0000 mon.smithi121 (mon.0) 713 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:50:02.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:00.228462+0000 mgr.smithi153.olhmgd (mgr.24421) 22 : cluster [DBG] pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 996 op/s 2024-04-23T00:50:02.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:01.407805+0000 mon.smithi121 (mon.0) 710 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:02.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:01.413554+0000 mon.smithi121 (mon.0) 711 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:02.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:01.419205+0000 mon.smithi121 (mon.0) 712 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:02.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:01.440755+0000 mon.smithi153 (mon.1) 98 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:50:02.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:01.441142+0000 mon.smithi121 (mon.0) 713 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-23T00:50:02.663 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104126 13.24 MB/sec execute 11 sec latency 248.333 ms 2024-04-23T00:50:03.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104768 13.08 MB/sec execute 12 sec latency 35.853 ms 2024-04-23T00:50:03.802 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:02.228759+0000 mgr.smithi153.olhmgd (mgr.24421) 23 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 671 op/s 2024-04-23T00:50:03.803 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:02.425346+0000 mon.smithi121 (mon.0) 714 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:50:03.803 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:02.425404+0000 mon.smithi121 (mon.0) 715 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-04-23T00:50:03.803 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:02.428034+0000 mon.smithi153 (mon.1) 99 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-23T00:50:03.803 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:02.429089+0000 mon.smithi121 (mon.0) 716 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-23T00:50:03.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:02.228759+0000 mgr.smithi153.olhmgd (mgr.24421) 23 : cluster [DBG] pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 671 op/s 2024-04-23T00:50:03.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:02.425346+0000 mon.smithi121 (mon.0) 714 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool create", "pool": ".nfs", "yes_i_really_mean_it": true}]': finished 2024-04-23T00:50:03.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:02.425404+0000 mon.smithi121 (mon.0) 715 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-04-23T00:50:03.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:02.428034+0000 mon.smithi153 (mon.1) 99 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-23T00:50:03.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:02.429089+0000 mon.smithi121 (mon.0) 716 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:03.423696+0000 mon.smithi121 (mon.0) 717 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:03.423861+0000 mon.smithi121 (mon.0) 718 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.438717+0000 mgr.smithi153.olhmgd (mgr.24421) 24 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.469823+0000 mgr.smithi153.olhmgd (mgr.24421) 25 : cephadm [INF] Found 1 exports for legacy nfs.foo 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.469945+0000 mgr.smithi153.olhmgd (mgr.24421) 26 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.smithi121', 'nfs.foo.smithi153'] 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.470019+0000 mgr.smithi153.olhmgd (mgr.24421) 27 : cephadm [INF] Remove daemons nfs.foo.smithi121 nfs.foo.smithi153 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.470483+0000 mgr.smithi153.olhmgd (mgr.24421) 28 : cephadm [INF] Removing daemon nfs.foo.smithi121 from smithi121 -- ports [2049] 2024-04-23T00:50:04.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:03.470783+0000 mgr.smithi153.olhmgd (mgr.24421) 29 : cephadm [INF] Removing daemon nfs.foo.smithi153 from smithi153 -- ports [2049] 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:03.423696+0000 mon.smithi121 (mon.0) 717 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:03.423861+0000 mon.smithi121 (mon.0) 718 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.438717+0000 mgr.smithi153.olhmgd (mgr.24421) 24 : cephadm [INF] Migrating nfs.foo from legacy pool nfs-ganesha namespace foo 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.469823+0000 mgr.smithi153.olhmgd (mgr.24421) 25 : cephadm [INF] Found 1 exports for legacy nfs.foo 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.469945+0000 mgr.smithi153.olhmgd (mgr.24421) 26 : cephadm [INF] Removing old nfs.foo daemons ['nfs.foo.smithi121', 'nfs.foo.smithi153'] 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.470019+0000 mgr.smithi153.olhmgd (mgr.24421) 27 : cephadm [INF] Remove daemons nfs.foo.smithi121 nfs.foo.smithi153 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.470483+0000 mgr.smithi153.olhmgd (mgr.24421) 28 : cephadm [INF] Removing daemon nfs.foo.smithi121 from smithi121 -- ports [2049] 2024-04-23T00:50:04.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:03.470783+0000 mgr.smithi153.olhmgd (mgr.24421) 29 : cephadm [INF] Removing daemon nfs.foo.smithi153 from smithi153 -- ports [2049] 2024-04-23T00:50:04.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 12.20 MB/sec execute 13 sec latency 725.976 ms 2024-04-23T00:50:05.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 11.33 MB/sec execute 14 sec latency 1726.041 ms 2024-04-23T00:50:05.770 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:04.229237+0000 mgr.smithi153.olhmgd (mgr.24421) 30 : cluster [DBG] pgmap v18: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 956 op/s 2024-04-23T00:50:05.770 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:04.438935+0000 mon.smithi121 (mon.0) 719 : cluster [DBG] mgrmap e30: smithi153.olhmgd(active, since 22s), standbys: smithi121.otezso 2024-04-23T00:50:05.770 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:04.438968+0000 mon.smithi121 (mon.0) 720 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-04-23T00:50:05.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:04.229237+0000 mgr.smithi153.olhmgd (mgr.24421) 30 : cluster [DBG] pgmap v18: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 956 op/s 2024-04-23T00:50:05.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:04.438935+0000 mon.smithi121 (mon.0) 719 : cluster [DBG] mgrmap e30: smithi153.olhmgd(active, since 22s), standbys: smithi121.otezso 2024-04-23T00:50:05.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:04.438968+0000 mon.smithi121 (mon.0) 720 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-04-23T00:50:06.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.990267+0000 mon.smithi153 (mon.1) 100 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.990884+0000 mon.smithi121 (mon.0) 721 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.994412+0000 mon.smithi121 (mon.0) 722 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]': finished 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.996478+0000 mon.smithi153 (mon.1) 101 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.996803+0000 mon.smithi121 (mon.0) 723 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:05.999156+0000 mon.smithi121 (mon.0) 724 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]': finished 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.243043+0000 mon.smithi153 (mon.1) 102 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.243349+0000 mon.smithi121 (mon.0) 725 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.246236+0000 mon.smithi121 (mon.0) 726 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]': finished 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.252919+0000 mon.smithi153 (mon.1) 103 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.253254+0000 mon.smithi121 (mon.0) 727 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]: dispatch 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.255570+0000 mon.smithi121 (mon.0) 728 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]': finished 2024-04-23T00:50:06.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.259663+0000 mon.smithi121 (mon.0) 729 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.260329+0000 mon.smithi153 (mon.1) 104 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.284629+0000 mon.smithi153 (mon.1) 105 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.284986+0000 mon.smithi121 (mon.0) 730 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.287839+0000 mon.smithi121 (mon.0) 731 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.297028+0000 mon.smithi121 (mon.0) 732 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.304014+0000 mon.smithi121 (mon.0) 733 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.308827+0000 mon.smithi121 (mon.0) 734 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.318065+0000 mon.smithi121 (mon.0) 735 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.326986+0000 mon.smithi121 (mon.0) 736 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.338175+0000 mon.smithi121 (mon.0) 737 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.339175+0000 mon.smithi153 (mon.1) 106 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:06.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.339659+0000 mon.smithi121 (mon.0) 738 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.342458+0000 mon.smithi121 (mon.0) 739 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.343356+0000 mon.smithi153 (mon.1) 107 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.343670+0000 mon.smithi121 (mon.0) 740 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.346063+0000 mon.smithi121 (mon.0) 741 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.347314+0000 mon.smithi153 (mon.1) 108 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.391923+0000 mon.smithi153 (mon.1) 109 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.392281+0000 mon.smithi121 (mon.0) 742 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:06.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.394855+0000 mon.smithi121 (mon.0) 743 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T00:50:06.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 10.58 MB/sec execute 15 sec latency 2726.410 ms 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.990267+0000 mon.smithi153 (mon.1) 100 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]: dispatch 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.990884+0000 mon.smithi121 (mon.0) 721 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]: dispatch 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.994412+0000 mon.smithi121 (mon.0) 722 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121"}]': finished 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.996478+0000 mon.smithi153 (mon.1) 101 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]: dispatch 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.996803+0000 mon.smithi121 (mon.0) 723 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]: dispatch 2024-04-23T00:50:06.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:05.999156+0000 mon.smithi121 (mon.0) 724 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi121-rgw"}]': finished 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.243043+0000 mon.smithi153 (mon.1) 102 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.243349+0000 mon.smithi121 (mon.0) 725 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.246236+0000 mon.smithi121 (mon.0) 726 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153"}]': finished 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.252919+0000 mon.smithi153 (mon.1) 103 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.253254+0000 mon.smithi121 (mon.0) 727 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.255570+0000 mon.smithi121 (mon.0) 728 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.nfs.foo.smithi153-rgw"}]': finished 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.259663+0000 mon.smithi121 (mon.0) 729 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.260329+0000 mon.smithi153 (mon.1) 104 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.284629+0000 mon.smithi153 (mon.1) 105 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.284986+0000 mon.smithi121 (mon.0) 730 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.287839+0000 mon.smithi121 (mon.0) 731 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.297028+0000 mon.smithi121 (mon.0) 732 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.304014+0000 mon.smithi121 (mon.0) 733 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.308827+0000 mon.smithi121 (mon.0) 734 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.318065+0000 mon.smithi121 (mon.0) 735 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.326986+0000 mon.smithi121 (mon.0) 736 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.338175+0000 mon.smithi121 (mon.0) 737 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.339175+0000 mon.smithi153 (mon.1) 106 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.339659+0000 mon.smithi121 (mon.0) 738 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.342458+0000 mon.smithi121 (mon.0) 739 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.343356+0000 mon.smithi153 (mon.1) 107 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.343670+0000 mon.smithi121 (mon.0) 740 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.346063+0000 mon.smithi121 (mon.0) 741 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.347314+0000 mon.smithi153 (mon.1) 108 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.391923+0000 mon.smithi153 (mon.1) 109 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.392281+0000 mon.smithi121 (mon.0) 742 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:06.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.394855+0000 mon.smithi121 (mon.0) 743 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T00:50:07.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:05.989847+0000 mgr.smithi153.olhmgd (mgr.24421) 31 : cephadm [INF] Removing key for client.nfs.foo.smithi121 2024-04-23T00:50:07.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:05.996167+0000 mgr.smithi153.olhmgd (mgr.24421) 32 : cephadm [INF] Removing key for client.nfs.foo.smithi121-rgw 2024-04-23T00:50:07.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:06.229675+0000 mgr.smithi153.olhmgd (mgr.24421) 33 : cluster [DBG] pgmap v20: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-23T00:50:07.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.242762+0000 mgr.smithi153.olhmgd (mgr.24421) 34 : cephadm [INF] Removing key for client.nfs.foo.smithi153 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.252671+0000 mgr.smithi153.olhmgd (mgr.24421) 35 : cephadm [INF] Removing key for client.nfs.foo.smithi153-rgw 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.260852+0000 mgr.smithi153.olhmgd (mgr.24421) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.293058+0000 mgr.smithi153.olhmgd (mgr.24421) 37 : cephadm [INF] Done migrating nfs.foo 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.293117+0000 mgr.smithi153.olhmgd (mgr.24421) 38 : cephadm [INF] Done migrating all NFS services 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.338922+0000 mgr.smithi153.olhmgd (mgr.24421) 39 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi121.aanwei 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.343178+0000 mgr.smithi153.olhmgd (mgr.24421) 40 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.449434+0000 mgr.smithi153.olhmgd (mgr.24421) 41 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.449523+0000 mgr.smithi153.olhmgd (mgr.24421) 42 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.449825+0000 mon.smithi153 (mon.1) 110 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.450357+0000 mon.smithi121 (mon.0) 744 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.455500+0000 mon.smithi121 (mon.0) 745 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.457052+0000 mgr.smithi153.olhmgd (mgr.24421) 43 : cephadm [WRN] Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:06.463662+0000 mon.smithi153 (mon.1) 111 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:06.464800+0000 mgr.smithi153.olhmgd (mgr.24421) 44 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T00:50:07.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 9.91 MB/sec execute 16 sec latency 3726.457 ms 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:05.989847+0000 mgr.smithi153.olhmgd (mgr.24421) 31 : cephadm [INF] Removing key for client.nfs.foo.smithi121 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:05.996167+0000 mgr.smithi153.olhmgd (mgr.24421) 32 : cephadm [INF] Removing key for client.nfs.foo.smithi121-rgw 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:06.229675+0000 mgr.smithi153.olhmgd (mgr.24421) 33 : cluster [DBG] pgmap v20: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.242762+0000 mgr.smithi153.olhmgd (mgr.24421) 34 : cephadm [INF] Removing key for client.nfs.foo.smithi153 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.252671+0000 mgr.smithi153.olhmgd (mgr.24421) 35 : cephadm [INF] Removing key for client.nfs.foo.smithi153-rgw 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.260852+0000 mgr.smithi153.olhmgd (mgr.24421) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "nfs export apply", "cluster_id": "foo"}]: dispatch 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.293058+0000 mgr.smithi153.olhmgd (mgr.24421) 37 : cephadm [INF] Done migrating nfs.foo 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.293117+0000 mgr.smithi153.olhmgd (mgr.24421) 38 : cephadm [INF] Done migrating all NFS services 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.338922+0000 mgr.smithi153.olhmgd (mgr.24421) 39 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi121.aanwei 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.343178+0000 mgr.smithi153.olhmgd (mgr.24421) 40 : cephadm [INF] Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.449434+0000 mgr.smithi153.olhmgd (mgr.24421) 41 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:50:07.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.449523+0000 mgr.smithi153.olhmgd (mgr.24421) 42 : cephadm [INF] Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.449825+0000 mon.smithi153 (mon.1) 110 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.450357+0000 mon.smithi121 (mon.0) 744 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.455500+0000 mon.smithi121 (mon.0) 745 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.457052+0000 mgr.smithi153.olhmgd (mgr.24421) 43 : cephadm [WRN] Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:06.463662+0000 mon.smithi153 (mon.1) 111 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:06.464800+0000 mgr.smithi153.olhmgd (mgr.24421) 44 : cephadm [INF] Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T00:50:08.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 9.33 MB/sec execute 17 sec latency 4726.544 ms 2024-04-23T00:50:09.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:08.230317+0000 mgr.smithi153.olhmgd (mgr.24421) 45 : cluster [DBG] pgmap v21: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-23T00:50:09.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.099391+0000 mon.smithi121 (mon.0) 746 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.105601+0000 mon.smithi121 (mon.0) 747 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.137199+0000 mon.smithi121 (mon.0) 748 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.138887+0000 mon.smithi153 (mon.1) 112 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.139186+0000 mon.smithi121 (mon.0) 749 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.141652+0000 mon.smithi121 (mon.0) 750 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.143342+0000 mon.smithi153 (mon.1) 113 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:09.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.143583+0000 mon.smithi121 (mon.0) 751 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:09.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:08.230317+0000 mgr.smithi153.olhmgd (mgr.24421) 45 : cluster [DBG] pgmap v21: 129 pgs: 32 creating+peering, 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.099391+0000 mon.smithi121 (mon.0) 746 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.105601+0000 mon.smithi121 (mon.0) 747 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.137199+0000 mon.smithi121 (mon.0) 748 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.138887+0000 mon.smithi153 (mon.1) 112 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.139186+0000 mon.smithi121 (mon.0) 749 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.141652+0000 mon.smithi121 (mon.0) 750 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.143342+0000 mon.smithi153 (mon.1) 113 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.143583+0000 mon.smithi121 (mon.0) 751 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T00:50:09.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 8.81 MB/sec execute 18 sec latency 5726.621 ms 2024-04-23T00:50:10.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:09.138654+0000 mgr.smithi153.olhmgd (mgr.24421) 46 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T00:50:10.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:09.143058+0000 mgr.smithi153.olhmgd (mgr.24421) 47 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T00:50:10.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.146014+0000 mon.smithi121 (mon.0) 752 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T00:50:10.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:09.150495+0000 mon.smithi153 (mon.1) 114 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:10.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:09.155600+0000 mon.smithi121 (mon.0) 753 : cluster [DBG] mgrmap e31: smithi153.olhmgd(active, since 27s), standbys: smithi121.otezso 2024-04-23T00:50:10.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:09.138654+0000 mgr.smithi153.olhmgd (mgr.24421) 46 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T00:50:10.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:09.143058+0000 mgr.smithi153.olhmgd (mgr.24421) 47 : cephadm [INF] Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T00:50:10.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.146014+0000 mon.smithi121 (mon.0) 752 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T00:50:10.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:09.150495+0000 mon.smithi153 (mon.1) 114 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:10.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:09.155600+0000 mon.smithi121 (mon.0) 753 : cluster [DBG] mgrmap e31: smithi153.olhmgd(active, since 27s), standbys: smithi121.otezso 2024-04-23T00:50:10.664 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 8.35 MB/sec execute 19 sec latency 6726.710 ms 2024-04-23T00:50:11.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:10.231154+0000 mgr.smithi153.olhmgd (mgr.24421) 48 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 315 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 573 op/s 2024-04-23T00:50:11.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:10.231154+0000 mgr.smithi153.olhmgd (mgr.24421) 48 : cluster [DBG] pgmap v22: 129 pgs: 129 active+clean; 315 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 573 op/s 2024-04-23T00:50:11.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 7.93 MB/sec execute 20 sec latency 7726.805 ms 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.006272+0000 mon.smithi153 (mon.1) 115 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.006678+0000 mon.smithi121 (mon.0) 754 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.011203+0000 mon.smithi121 (mon.0) 755 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.069718+0000 mon.smithi153 (mon.1) 116 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.070209+0000 mon.smithi121 (mon.0) 756 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.073781+0000 mon.smithi121 (mon.0) 757 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:50:12.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.075038+0000 mon.smithi153 (mon.1) 117 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:12.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.006272+0000 mon.smithi153 (mon.1) 115 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:12.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.006678+0000 mon.smithi121 (mon.0) 754 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T00:50:12.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.011203+0000 mon.smithi121 (mon.0) 755 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T00:50:12.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.069718+0000 mon.smithi153 (mon.1) 116 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:12.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.070209+0000 mon.smithi121 (mon.0) 756 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T00:50:12.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.073781+0000 mon.smithi121 (mon.0) 757 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-23T00:50:12.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.075038+0000 mon.smithi153 (mon.1) 117 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:12.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 7.55 MB/sec execute 21 sec latency 8726.905 ms 2024-04-23T00:50:13.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:12.069229+0000 mgr.smithi153.olhmgd (mgr.24421) 49 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:50:13.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:12.069346+0000 mgr.smithi153.olhmgd (mgr.24421) 50 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T00:50:13.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:12.074532+0000 mgr.smithi153.olhmgd (mgr.24421) 51 : cephadm [WRN] Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T00:50:13.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:12.076127+0000 mgr.smithi153.olhmgd (mgr.24421) 52 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T00:50:13.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:12.231602+0000 mgr.smithi153.olhmgd (mgr.24421) 53 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 315 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 580 KiB/s wr, 61 op/s 2024-04-23T00:50:13.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:12.394061+0000 mon.smithi153 (mon.1) 118 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:12.069229+0000 mgr.smithi153.olhmgd (mgr.24421) 49 : cephadm [INF] Rados config object exists: conf-nfs.foo 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:12.069346+0000 mgr.smithi153.olhmgd (mgr.24421) 50 : cephadm [INF] Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:12.074532+0000 mgr.smithi153.olhmgd (mgr.24421) 51 : cephadm [WRN] Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:12.076127+0000 mgr.smithi153.olhmgd (mgr.24421) 52 : cephadm [INF] Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:12.231602+0000 mgr.smithi153.olhmgd (mgr.24421) 53 : cluster [DBG] pgmap v23: 129 pgs: 129 active+clean; 315 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 580 KiB/s wr, 61 op/s 2024-04-23T00:50:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:12.394061+0000 mon.smithi153 (mon.1) 118 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:13.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 7.21 MB/sec execute 22 sec latency 9726.950 ms 2024-04-23T00:50:14.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 6.90 MB/sec execute 23 sec latency 10726.983 ms 2024-04-23T00:50:15.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 6.61 MB/sec execute 24 sec latency 11727.061 ms 2024-04-23T00:50:15.822 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:14.232239+0000 mgr.smithi153.olhmgd (mgr.24421) 54 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 517 KiB/s wr, 58 op/s 2024-04-23T00:50:15.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:14.769099+0000 mon.smithi121 (mon.0) 758 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:15.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:14.774545+0000 mon.smithi121 (mon.0) 759 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:15.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:14.805425+0000 mon.smithi121 (mon.0) 760 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:15.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:14.810684+0000 mon.smithi121 (mon.0) 761 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:15.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:14.940111+0000 mon.smithi153 (mon.1) 119 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:14.232239+0000 mgr.smithi153.olhmgd (mgr.24421) 54 : cluster [DBG] pgmap v24: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 517 KiB/s wr, 58 op/s 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:14.769099+0000 mon.smithi121 (mon.0) 758 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:14.774545+0000 mon.smithi121 (mon.0) 759 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:14.805425+0000 mon.smithi121 (mon.0) 760 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:14.810684+0000 mon.smithi121 (mon.0) 761 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:16.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:14.940111+0000 mon.smithi153 (mon.1) 119 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T00:50:16.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 6.35 MB/sec execute 25 sec latency 12727.101 ms 2024-04-23T00:50:17.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 6.10 MB/sec execute 26 sec latency 13727.129 ms 2024-04-23T00:50:18.031 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:16.232716+0000 mgr.smithi153.olhmgd (mgr.24421) 55 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 438 KiB/s wr, 49 op/s 2024-04-23T00:50:18.031 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:17.395028+0000 mon.smithi121 (mon.0) 762 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:18.074 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:16.232716+0000 mgr.smithi153.olhmgd (mgr.24421) 55 : cluster [DBG] pgmap v25: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 438 KiB/s wr, 49 op/s 2024-04-23T00:50:18.074 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:17.395028+0000 mon.smithi121 (mon.0) 762 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:18.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 5.88 MB/sec execute 27 sec latency 14727.170 ms 2024-04-23T00:50:19.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:18.233094+0000 mgr.smithi153.olhmgd (mgr.24421) 56 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 431 KiB/s wr, 49 op/s 2024-04-23T00:50:19.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:19.118292+0000 mon.smithi121 (mon.0) 763 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:19.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:19.123745+0000 mon.smithi121 (mon.0) 764 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:19.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:18.233094+0000 mgr.smithi153.olhmgd (mgr.24421) 56 : cluster [DBG] pgmap v26: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 431 KiB/s wr, 49 op/s 2024-04-23T00:50:19.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:19.118292+0000 mon.smithi121 (mon.0) 763 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:19.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:19.123745+0000 mon.smithi121 (mon.0) 764 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:19.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 5.67 MB/sec execute 28 sec latency 15727.238 ms 2024-04-23T00:50:20.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 5.47 MB/sec execute 29 sec latency 16727.291 ms 2024-04-23T00:50:21.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:20.234004+0000 mgr.smithi153.olhmgd (mgr.24421) 57 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 432 KiB/s wr, 52 op/s 2024-04-23T00:50:21.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:20.234004+0000 mgr.smithi153.olhmgd (mgr.24421) 57 : cluster [DBG] pgmap v27: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 432 KiB/s wr, 52 op/s 2024-04-23T00:50:21.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 5.29 MB/sec execute 30 sec latency 17727.317 ms 2024-04-23T00:50:22.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.475201+0000 mon.smithi121 (mon.0) 765 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.481820+0000 mon.smithi121 (mon.0) 766 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.483709+0000 mon.smithi153 (mon.1) 120 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:22.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.484458+0000 mon.smithi153 (mon.1) 121 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T00:50:22.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:21.485126+0000 mgr.smithi153.olhmgd (mgr.24421) 58 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T00:50:22.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.488301+0000 mon.smithi121 (mon.0) 767 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.496142+0000 mon.smithi121 (mon.0) 768 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.629705+0000 mon.smithi121 (mon.0) 769 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.632925+0000 mon.smithi121 (mon.0) 770 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.635979+0000 mon.smithi121 (mon.0) 771 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:21.639184+0000 mon.smithi121 (mon.0) 772 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:21.640140+0000 mgr.smithi153.olhmgd (mgr.24421) 59 : cephadm [INF] Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T00:50:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:50:21.647336+0000 mgr.smithi153.olhmgd (mgr.24421) 60 : cephadm [INF] Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T00:50:22.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 5.12 MB/sec execute 31 sec latency 18727.363 ms 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.475201+0000 mon.smithi121 (mon.0) 765 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.481820+0000 mon.smithi121 (mon.0) 766 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.483709+0000 mon.smithi153 (mon.1) 120 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.484458+0000 mon.smithi153 (mon.1) 121 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:21.485126+0000 mgr.smithi153.olhmgd (mgr.24421) 58 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.488301+0000 mon.smithi121 (mon.0) 767 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.496142+0000 mon.smithi121 (mon.0) 768 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.629705+0000 mon.smithi121 (mon.0) 769 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.632925+0000 mon.smithi121 (mon.0) 770 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.635979+0000 mon.smithi121 (mon.0) 771 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:21.639184+0000 mon.smithi121 (mon.0) 772 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:22.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:21.640140+0000 mgr.smithi153.olhmgd (mgr.24421) 59 : cephadm [INF] Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T00:50:22.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:50:21.647336+0000 mgr.smithi153.olhmgd (mgr.24421) 60 : cephadm [INF] Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T00:50:23.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.96 MB/sec execute 32 sec latency 19727.402 ms 2024-04-23T00:50:24.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:22.234429+0000 mgr.smithi153.olhmgd (mgr.24421) 61 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.6 KiB/s wr, 7 op/s 2024-04-23T00:50:24.127 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:22.234429+0000 mgr.smithi153.olhmgd (mgr.24421) 61 : cluster [DBG] pgmap v28: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.6 KiB/s wr, 7 op/s 2024-04-23T00:50:24.665 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.81 MB/sec execute 33 sec latency 20727.427 ms 2024-04-23T00:50:25.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.67 MB/sec execute 34 sec latency 21727.520 ms 2024-04-23T00:50:26.072 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:24.235159+0000 mgr.smithi153.olhmgd (mgr.24421) 62 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 6.4 KiB/s wr, 7 op/s 2024-04-23T00:50:26.118 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:24.235159+0000 mgr.smithi153.olhmgd (mgr.24421) 62 : cluster [DBG] pgmap v29: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 6.4 KiB/s wr, 7 op/s 2024-04-23T00:50:26.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.53 MB/sec execute 35 sec latency 22727.617 ms 2024-04-23T00:50:27.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.41 MB/sec execute 36 sec latency 23727.712 ms 2024-04-23T00:50:27.716 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:26.235675+0000 mgr.smithi153.olhmgd (mgr.24421) 63 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:27.716 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:27.394375+0000 mon.smithi153 (mon.1) 122 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:28.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:26.235675+0000 mgr.smithi153.olhmgd (mgr.24421) 63 : cluster [DBG] pgmap v30: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:28.073 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:27.394375+0000 mon.smithi153 (mon.1) 122 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:28.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.29 MB/sec execute 37 sec latency 24727.782 ms 2024-04-23T00:50:29.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:28.236088+0000 mgr.smithi153.olhmgd (mgr.24421) 64 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:29.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:28.236088+0000 mgr.smithi153.olhmgd (mgr.24421) 64 : cluster [DBG] pgmap v31: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:29.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.17 MB/sec execute 38 sec latency 25727.874 ms 2024-04-23T00:50:30.222 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:50:30.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 4.07 MB/sec execute 39 sec latency 26727.939 ms 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (7m) 9s ago 8m 14.9M - 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (8m) 9s ago 8m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (7m) 11s ago 7m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (6m) 9s ago 8m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (2m) 9s ago 2m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (3m) 11s ago 3m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (10m) 9s ago 10m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (58s) 11s ago 7m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (10m) 9s ago 10m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (7m) 11s ago 7m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:50:30.847 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (22s) 9s ago 21s 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (16s) 11s ago 16s 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (8m) 9s ago 8m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (7m) 11s ago 7m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (6m) 9s ago 6m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (6m) 9s ago 6m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (6m) 9s ago 6m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (5m) 9s ago 5m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (5m) 11s ago 5m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (5m) 11s ago 5m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (4m) 11s ago 4m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (4m) 11s ago 4m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:50:30.848 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (6m) 9s ago 7m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:50:31.219 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:50:31.220 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:50:31.532 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:50:31.532 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:50:31.532 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:50:31.533 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:50:31.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:30.209695+0000 mgr.smithi153.olhmgd (mgr.24421) 65 : audit [DBG] from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:31.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:30.236770+0000 mgr.smithi153.olhmgd (mgr.24421) 66 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:31.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:30.528335+0000 mgr.smithi153.olhmgd (mgr.24421) 67 : audit [DBG] from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:31.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:31.218653+0000 mon.smithi121 (mon.0) 773 : audit [DBG] from='client.? 172.21.15.121:0/2931261751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:50:31.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:30.209695+0000 mgr.smithi153.olhmgd (mgr.24421) 65 : audit [DBG] from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:31.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:30.236770+0000 mgr.smithi153.olhmgd (mgr.24421) 66 : cluster [DBG] pgmap v32: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-23T00:50:31.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:30.528335+0000 mgr.smithi153.olhmgd (mgr.24421) 67 : audit [DBG] from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:31.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:31.218653+0000 mon.smithi121 (mon.0) 773 : audit [DBG] from='client.? 172.21.15.121:0/2931261751' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:50:31.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.97 MB/sec execute 40 sec latency 27727.971 ms 2024-04-23T00:50:31.908 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:50:32.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:30.835348+0000 mgr.smithi153.olhmgd (mgr.24421) 68 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:32.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:31.531458+0000 mgr.smithi153.olhmgd (mgr.24421) 69 : audit [DBG] from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:32.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:31.907919+0000 mon.smithi121 (mon.0) 774 : audit [DBG] from='client.? 172.21.15.121:0/2699234490' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:50:32.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:30.835348+0000 mgr.smithi153.olhmgd (mgr.24421) 68 : audit [DBG] from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:32.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:31.531458+0000 mgr.smithi153.olhmgd (mgr.24421) 69 : audit [DBG] from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:50:32.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:31.907919+0000 mon.smithi121 (mon.0) 774 : audit [DBG] from='client.? 172.21.15.121:0/2699234490' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:50:32.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.87 MB/sec execute 41 sec latency 28728.035 ms 2024-04-23T00:50:33.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:32.237303+0000 mgr.smithi153.olhmgd (mgr.24421) 70 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 853 B/s wr, 0 op/s 2024-04-23T00:50:33.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:32.237303+0000 mgr.smithi153.olhmgd (mgr.24421) 70 : cluster [DBG] pgmap v33: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 853 B/s wr, 0 op/s 2024-04-23T00:50:33.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.78 MB/sec execute 42 sec latency 29728.111 ms 2024-04-23T00:50:34.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.69 MB/sec execute 43 sec latency 30728.155 ms 2024-04-23T00:50:35.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:34.238234+0000 mgr.smithi153.olhmgd (mgr.24421) 71 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 853 B/s wr, 0 op/s 2024-04-23T00:50:35.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:34.238234+0000 mgr.smithi153.olhmgd (mgr.24421) 71 : cluster [DBG] pgmap v34: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 853 B/s wr, 0 op/s 2024-04-23T00:50:35.666 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.61 MB/sec execute 44 sec latency 31728.237 ms 2024-04-23T00:50:36.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.53 MB/sec execute 45 sec latency 32728.338 ms 2024-04-23T00:50:37.572 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:36.238706+0000 mgr.smithi153.olhmgd (mgr.24421) 72 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:37.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:36.238706+0000 mgr.smithi153.olhmgd (mgr.24421) 72 : cluster [DBG] pgmap v35: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:37.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.45 MB/sec execute 46 sec latency 33728.423 ms 2024-04-23T00:50:38.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.38 MB/sec execute 47 sec latency 34728.517 ms 2024-04-23T00:50:39.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:38.239221+0000 mgr.smithi153.olhmgd (mgr.24421) 73 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:39.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:38.239221+0000 mgr.smithi153.olhmgd (mgr.24421) 73 : cluster [DBG] pgmap v36: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:39.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.31 MB/sec execute 48 sec latency 35728.610 ms 2024-04-23T00:50:40.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.24 MB/sec execute 49 sec latency 36728.703 ms 2024-04-23T00:50:41.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:40.239892+0000 mgr.smithi153.olhmgd (mgr.24421) 74 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:41.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:40.239892+0000 mgr.smithi153.olhmgd (mgr.24421) 74 : cluster [DBG] pgmap v37: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:41.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.17 MB/sec execute 50 sec latency 37728.801 ms 2024-04-23T00:50:42.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.11 MB/sec execute 51 sec latency 38728.907 ms 2024-04-23T00:50:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:42.240426+0000 mgr.smithi153.olhmgd (mgr.24421) 75 : cluster [DBG] pgmap v38: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:42.394647+0000 mon.smithi153 (mon.1) 123 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:42.240426+0000 mgr.smithi153.olhmgd (mgr.24421) 75 : cluster [DBG] pgmap v38: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:43.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:42.394647+0000 mon.smithi153 (mon.1) 123 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:43.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 3.05 MB/sec execute 52 sec latency 39728.994 ms 2024-04-23T00:50:44.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:44.017139+0000 mds.foofs.smithi153.uunfav (mds.0) 1 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.908949 secs 2024-04-23T00:50:44.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:44.017145+0000 mds.foofs.smithi153.uunfav (mds.0) 2 : cluster [WRN] slow request 30.908949 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:50:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:44.017139+0000 mds.foofs.smithi153.uunfav (mds.0) 1 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.908949 secs 2024-04-23T00:50:44.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:44.017145+0000 mds.foofs.smithi153.uunfav (mds.0) 2 : cluster [WRN] slow request 30.908949 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:50:44.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.99 MB/sec execute 53 sec latency 40729.066 ms 2024-04-23T00:50:45.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:44.241170+0000 mgr.smithi153.olhmgd (mgr.24421) 76 : cluster [DBG] pgmap v39: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:45.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:44.241170+0000 mgr.smithi153.olhmgd (mgr.24421) 76 : cluster [DBG] pgmap v39: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:45.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.94 MB/sec execute 54 sec latency 41729.141 ms 2024-04-23T00:50:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:46.305355+0000 mon.smithi121 (mon.0) 775 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2024-04-23T00:50:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:46.311446+0000 mon.smithi121 (mon.0) 776 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:50:46.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:46.311508+0000 mon.smithi121 (mon.0) 777 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:50:46.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.88 MB/sec execute 55 sec latency 42729.237 ms 2024-04-23T00:50:46.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:46.305355+0000 mon.smithi121 (mon.0) 775 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2024-04-23T00:50:46.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:46.311446+0000 mon.smithi121 (mon.0) 776 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:50:46.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:46.311508+0000 mon.smithi121 (mon.0) 777 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:50:47.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:46.241661+0000 mgr.smithi153.olhmgd (mgr.24421) 77 : cluster [DBG] pgmap v40: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:47.667 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.83 MB/sec execute 56 sec latency 43729.305 ms 2024-04-23T00:50:47.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:46.241661+0000 mgr.smithi153.olhmgd (mgr.24421) 77 : cluster [DBG] pgmap v40: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:48.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.78 MB/sec execute 57 sec latency 44729.376 ms 2024-04-23T00:50:49.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:48.242266+0000 mgr.smithi153.olhmgd (mgr.24421) 78 : cluster [DBG] pgmap v41: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:49.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:49.017239+0000 mds.foofs.smithi153.uunfav (mds.0) 3 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 35.909063 secs 2024-04-23T00:50:49.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:48.242266+0000 mgr.smithi153.olhmgd (mgr.24421) 78 : cluster [DBG] pgmap v41: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:49.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:49.017239+0000 mds.foofs.smithi153.uunfav (mds.0) 3 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 35.909063 secs 2024-04-23T00:50:49.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.74 MB/sec execute 58 sec latency 45729.416 ms 2024-04-23T00:50:50.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.69 MB/sec execute 59 sec latency 46729.547 ms 2024-04-23T00:50:51.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:50.242867+0000 mgr.smithi153.olhmgd (mgr.24421) 79 : cluster [DBG] pgmap v42: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:51.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:50.242867+0000 mgr.smithi153.olhmgd (mgr.24421) 79 : cluster [DBG] pgmap v42: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:51.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.64 MB/sec execute 60 sec latency 47729.631 ms 2024-04-23T00:50:52.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.60 MB/sec execute 61 sec latency 48729.724 ms 2024-04-23T00:50:53.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:52.243280+0000 mgr.smithi153.olhmgd (mgr.24421) 80 : cluster [DBG] pgmap v43: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:53.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:52.243280+0000 mgr.smithi153.olhmgd (mgr.24421) 80 : cluster [DBG] pgmap v43: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:53.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.56 MB/sec execute 62 sec latency 49729.800 ms 2024-04-23T00:50:54.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:54.017369+0000 mds.foofs.smithi153.uunfav (mds.0) 4 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 40.909179 secs 2024-04-23T00:50:54.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:54.017369+0000 mds.foofs.smithi153.uunfav (mds.0) 4 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 40.909179 secs 2024-04-23T00:50:54.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.52 MB/sec execute 63 sec latency 50729.837 ms 2024-04-23T00:50:55.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:54.244013+0000 mgr.smithi153.olhmgd (mgr.24421) 81 : cluster [DBG] pgmap v44: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:55.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:54.244013+0000 mgr.smithi153.olhmgd (mgr.24421) 81 : cluster [DBG] pgmap v44: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:55.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.48 MB/sec execute 64 sec latency 51729.952 ms 2024-04-23T00:50:56.668 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.44 MB/sec execute 65 sec latency 52730.052 ms 2024-04-23T00:50:57.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:56.244533+0000 mgr.smithi153.olhmgd (mgr.24421) 82 : cluster [DBG] pgmap v45: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:57.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:56.244533+0000 mgr.smithi153.olhmgd (mgr.24421) 82 : cluster [DBG] pgmap v45: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:50:57.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.40 MB/sec execute 66 sec latency 53730.153 ms 2024-04-23T00:50:58.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.37 MB/sec execute 67 sec latency 54730.248 ms 2024-04-23T00:50:58.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:57.400830+0000 mon.smithi121 (mon.0) 778 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:58.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:50:57.401933+0000 mon.smithi153 (mon.1) 124 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:58.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:57.400830+0000 mon.smithi121 (mon.0) 778 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:50:58.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:50:57.401933+0000 mon.smithi153 (mon.1) 124 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:50:59.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.33 MB/sec execute 68 sec latency 55730.363 ms 2024-04-23T00:50:59.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:58.245215+0000 mgr.smithi153.olhmgd (mgr.24421) 83 : cluster [DBG] pgmap v46: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:59.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:50:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:50:59.017448+0000 mds.foofs.smithi153.uunfav (mds.0) 5 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 45.909272 secs 2024-04-23T00:50:59.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:58.245215+0000 mgr.smithi153.olhmgd (mgr.24421) 83 : cluster [DBG] pgmap v46: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:50:59.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:50:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:50:59.017448+0000 mds.foofs.smithi153.uunfav (mds.0) 5 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 45.909272 secs 2024-04-23T00:51:00.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.30 MB/sec execute 69 sec latency 56730.446 ms 2024-04-23T00:51:01.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.27 MB/sec execute 70 sec latency 57730.557 ms 2024-04-23T00:51:01.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:00.245744+0000 mgr.smithi153.olhmgd (mgr.24421) 84 : cluster [DBG] pgmap v47: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:01.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:00.245744+0000 mgr.smithi153.olhmgd (mgr.24421) 84 : cluster [DBG] pgmap v47: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:02.245 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:51:02.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.23 MB/sec execute 71 sec latency 58730.623 ms 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (7m) 41s ago 9m 14.9M - 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (9m) 41s ago 9m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (8m) 43s ago 8m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (7m) 41s ago 8m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (3m) 41s ago 3m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (3m) 43s ago 3m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (10m) 41s ago 10m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (90s) 43s ago 8m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (10m) 41s ago 10m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:51:02.866 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (8m) 43s ago 8m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (54s) 41s ago 53s 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (48s) 43s ago 48s 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (8m) 41s ago 8m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (7m) 43s ago 7m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (7m) 41s ago 7m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (7m) 41s ago 7m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (6m) 41s ago 6m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (6m) 41s ago 6m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (6m) 43s ago 6m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (5m) 43s ago 5m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (5m) 43s ago 5m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (5m) 43s ago 5m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:51:02.867 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (7m) 41s ago 8m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:51:03.238 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:51:03.239 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:03.240 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:51:03.240 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:51:03.240 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:51:03.240 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:51:03.240 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:51:03.555 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:51:03.556 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:51:03.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.20 MB/sec execute 72 sec latency 59730.674 ms 2024-04-23T00:51:03.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:02.231812+0000 mgr.smithi153.olhmgd (mgr.24421) 85 : audit [DBG] from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:03.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:02.246257+0000 mgr.smithi153.olhmgd (mgr.24421) 86 : cluster [DBG] pgmap v48: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:03.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:02.547561+0000 mgr.smithi153.olhmgd (mgr.24421) 87 : audit [DBG] from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:03.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:03.237886+0000 mon.smithi121 (mon.0) 779 : audit [DBG] from='client.? 172.21.15.121:0/3674272487' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:51:03.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:02.231812+0000 mgr.smithi153.olhmgd (mgr.24421) 85 : audit [DBG] from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:03.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:02.246257+0000 mgr.smithi153.olhmgd (mgr.24421) 86 : cluster [DBG] pgmap v48: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:03.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:02.547561+0000 mgr.smithi153.olhmgd (mgr.24421) 87 : audit [DBG] from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:03.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:03.237886+0000 mon.smithi121 (mon.0) 779 : audit [DBG] from='client.? 172.21.15.121:0/3674272487' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:51:03.929 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 MDSs report slow requests 2024-04-23T00:51:03.929 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:51:03.929 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:51:04.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.17 MB/sec execute 73 sec latency 60730.771 ms 2024-04-23T00:51:04.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:02.854175+0000 mgr.smithi153.olhmgd (mgr.24421) 88 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:04.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:03.553944+0000 mgr.smithi153.olhmgd (mgr.24421) 89 : audit [DBG] from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:04.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:03.929044+0000 mon.smithi121 (mon.0) 780 : audit [DBG] from='client.? 172.21.15.121:0/3125285497' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:51:04.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:04.017579+0000 mds.foofs.smithi153.uunfav (mds.0) 6 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 50.909389 secs 2024-04-23T00:51:04.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:02.854175+0000 mgr.smithi153.olhmgd (mgr.24421) 88 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:04.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:03.553944+0000 mgr.smithi153.olhmgd (mgr.24421) 89 : audit [DBG] from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:04.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:03.929044+0000 mon.smithi121 (mon.0) 780 : audit [DBG] from='client.? 172.21.15.121:0/3125285497' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:51:04.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:04.017579+0000 mds.foofs.smithi153.uunfav (mds.0) 6 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 50.909389 secs 2024-04-23T00:51:05.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.14 MB/sec execute 74 sec latency 61730.865 ms 2024-04-23T00:51:05.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:04.246956+0000 mgr.smithi153.olhmgd (mgr.24421) 90 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:05.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:04.246956+0000 mgr.smithi153.olhmgd (mgr.24421) 90 : cluster [DBG] pgmap v49: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:06.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.12 MB/sec execute 75 sec latency 62730.966 ms 2024-04-23T00:51:07.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.09 MB/sec execute 76 sec latency 63731.050 ms 2024-04-23T00:51:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:06.247552+0000 mgr.smithi153.olhmgd (mgr.24421) 91 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:07.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:06.247552+0000 mgr.smithi153.olhmgd (mgr.24421) 91 : cluster [DBG] pgmap v50: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:08.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.06 MB/sec execute 77 sec latency 64731.087 ms 2024-04-23T00:51:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:08.248034+0000 mgr.smithi153.olhmgd (mgr.24421) 92 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:09.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:09.017678+0000 mds.foofs.smithi153.uunfav (mds.0) 7 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 55.909502 secs 2024-04-23T00:51:09.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:08.248034+0000 mgr.smithi153.olhmgd (mgr.24421) 92 : cluster [DBG] pgmap v51: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:09.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:09.017678+0000 mds.foofs.smithi153.uunfav (mds.0) 7 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 55.909502 secs 2024-04-23T00:51:09.669 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.03 MB/sec execute 78 sec latency 65731.161 ms 2024-04-23T00:51:10.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 2.01 MB/sec execute 79 sec latency 66731.222 ms 2024-04-23T00:51:11.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:10.248680+0000 mgr.smithi153.olhmgd (mgr.24421) 93 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:11.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:10.248680+0000 mgr.smithi153.olhmgd (mgr.24421) 93 : cluster [DBG] pgmap v52: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:11.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.98 MB/sec execute 80 sec latency 67731.314 ms 2024-04-23T00:51:12.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.96 MB/sec execute 81 sec latency 68731.416 ms 2024-04-23T00:51:13.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:12.249214+0000 mgr.smithi153.olhmgd (mgr.24421) 94 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:13.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:12.395363+0000 mon.smithi153 (mon.1) 125 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:13.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:12.249214+0000 mgr.smithi153.olhmgd (mgr.24421) 94 : cluster [DBG] pgmap v53: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:13.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:12.395363+0000 mon.smithi153 (mon.1) 125 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:13.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.93 MB/sec execute 82 sec latency 69731.527 ms 2024-04-23T00:51:14.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:13.318217+0000 mon.smithi121 (mon.0) 781 : cluster [DBG] mgrmap e32: smithi153.olhmgd(active, since 91s), standbys: smithi121.otezso 2024-04-23T00:51:14.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:14.017843+0000 mds.foofs.smithi153.uunfav (mds.0) 8 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 60.909639 secs 2024-04-23T00:51:14.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:14.017853+0000 mds.foofs.smithi153.uunfav (mds.0) 9 : cluster [WRN] slow request 60.909638 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:51:14.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:14.017895+0000 mds.foofs.smithi153.uunfav (mds.0) 10 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 60.909692 seconds ago 2024-04-23T00:51:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:13.318217+0000 mon.smithi121 (mon.0) 781 : cluster [DBG] mgrmap e32: smithi153.olhmgd(active, since 91s), standbys: smithi121.otezso 2024-04-23T00:51:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:14.017843+0000 mds.foofs.smithi153.uunfav (mds.0) 8 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 60.909639 secs 2024-04-23T00:51:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:14.017853+0000 mds.foofs.smithi153.uunfav (mds.0) 9 : cluster [WRN] slow request 60.909638 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:51:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:14.017895+0000 mds.foofs.smithi153.uunfav (mds.0) 10 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 60.909692 seconds ago 2024-04-23T00:51:14.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.91 MB/sec execute 83 sec latency 70731.574 ms 2024-04-23T00:51:15.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:14.250050+0000 mgr.smithi153.olhmgd (mgr.24421) 95 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:15.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:14.250050+0000 mgr.smithi153.olhmgd (mgr.24421) 95 : cluster [DBG] pgmap v54: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:15.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.89 MB/sec execute 84 sec latency 71731.618 ms 2024-04-23T00:51:16.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.87 MB/sec execute 85 sec latency 72731.734 ms 2024-04-23T00:51:17.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:16.250462+0000 mgr.smithi153.olhmgd (mgr.24421) 96 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:17.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.84 MB/sec execute 86 sec latency 73731.801 ms 2024-04-23T00:51:17.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:16.250462+0000 mgr.smithi153.olhmgd (mgr.24421) 96 : cluster [DBG] pgmap v55: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:18.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.82 MB/sec execute 87 sec latency 74731.881 ms 2024-04-23T00:51:19.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:18.250995+0000 mgr.smithi153.olhmgd (mgr.24421) 97 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:19.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:18.322618+0000 mon.smithi121 (mon.0) 782 : cluster [WRN] Health check failed: 1 clients failing to respond to capability release (MDS_CLIENT_LATE_RELEASE) 2024-04-23T00:51:19.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:18.327774+0000 mon.smithi121 (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:51:19.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:18.327856+0000 mon.smithi121 (mon.0) 784 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:51:19.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:19.017966+0000 mds.foofs.smithi153.uunfav (mds.0) 11 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 65.909775 secs 2024-04-23T00:51:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:18.250995+0000 mgr.smithi153.olhmgd (mgr.24421) 97 : cluster [DBG] pgmap v56: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:18.322618+0000 mon.smithi121 (mon.0) 782 : cluster [WRN] Health check failed: 1 clients failing to respond to capability release (MDS_CLIENT_LATE_RELEASE) 2024-04-23T00:51:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:18.327774+0000 mon.smithi121 (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:51:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:18.327856+0000 mon.smithi121 (mon.0) 784 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:51:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:19.017966+0000 mds.foofs.smithi153.uunfav (mds.0) 11 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 65.909775 secs 2024-04-23T00:51:19.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.80 MB/sec execute 88 sec latency 75731.933 ms 2024-04-23T00:51:20.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.78 MB/sec execute 89 sec latency 76731.971 ms 2024-04-23T00:51:21.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:20.251775+0000 mgr.smithi153.olhmgd (mgr.24421) 98 : cluster [DBG] pgmap v57: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:21.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:20.251775+0000 mgr.smithi153.olhmgd (mgr.24421) 98 : cluster [DBG] pgmap v57: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:21.670 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.76 MB/sec execute 90 sec latency 77732.013 ms 2024-04-23T00:51:22.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.74 MB/sec execute 91 sec latency 78732.074 ms 2024-04-23T00:51:23.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:22.252216+0000 mgr.smithi153.olhmgd (mgr.24421) 99 : cluster [DBG] pgmap v58: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:23.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:22.252216+0000 mgr.smithi153.olhmgd (mgr.24421) 99 : cluster [DBG] pgmap v58: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:23.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.72 MB/sec execute 92 sec latency 79732.181 ms 2024-04-23T00:51:24.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:24.018031+0000 mds.foofs.smithi153.uunfav (mds.0) 12 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 70.909840 secs 2024-04-23T00:51:24.627 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:24.018031+0000 mds.foofs.smithi153.uunfav (mds.0) 12 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 70.909840 secs 2024-04-23T00:51:24.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.71 MB/sec execute 93 sec latency 80732.229 ms 2024-04-23T00:51:25.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:24.252938+0000 mgr.smithi153.olhmgd (mgr.24421) 100 : cluster [DBG] pgmap v59: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:25.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:24.252938+0000 mgr.smithi153.olhmgd (mgr.24421) 100 : cluster [DBG] pgmap v59: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-04-23T00:51:25.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.69 MB/sec execute 94 sec latency 81732.313 ms 2024-04-23T00:51:26.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.67 MB/sec execute 95 sec latency 82732.416 ms 2024-04-23T00:51:27.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:26.253327+0000 mgr.smithi153.olhmgd (mgr.24421) 101 : cluster [DBG] pgmap v60: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:27.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:26.253327+0000 mgr.smithi153.olhmgd (mgr.24421) 101 : cluster [DBG] pgmap v60: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:27.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.65 MB/sec execute 96 sec latency 83732.496 ms 2024-04-23T00:51:28.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.64 MB/sec execute 97 sec latency 84732.608 ms 2024-04-23T00:51:28.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:27.399720+0000 mon.smithi121 (mon.0) 785 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:51:28.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:27.400953+0000 mon.smithi153 (mon.1) 126 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:28.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:27.399720+0000 mon.smithi121 (mon.0) 785 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:51:28.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:27.400953+0000 mon.smithi153 (mon.1) 126 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:29.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.62 MB/sec execute 98 sec latency 85732.672 ms 2024-04-23T00:51:29.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:28.253903+0000 mgr.smithi153.olhmgd (mgr.24421) 102 : cluster [DBG] pgmap v61: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:29.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:29.018127+0000 mds.foofs.smithi153.uunfav (mds.0) 13 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 75.909950 secs 2024-04-23T00:51:29.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:28.253903+0000 mgr.smithi153.olhmgd (mgr.24421) 102 : cluster [DBG] pgmap v61: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:29.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:29.018127+0000 mds.foofs.smithi153.uunfav (mds.0) 13 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 75.909950 secs 2024-04-23T00:51:30.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.60 MB/sec execute 99 sec latency 86732.773 ms 2024-04-23T00:51:31.671 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.59 MB/sec execute 100 sec latency 87732.854 ms 2024-04-23T00:51:31.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:30.254301+0000 mgr.smithi153.olhmgd (mgr.24421) 103 : cluster [DBG] pgmap v62: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:31.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:30.254301+0000 mgr.smithi153.olhmgd (mgr.24421) 103 : cluster [DBG] pgmap v62: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:32.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.57 MB/sec execute 101 sec latency 88732.897 ms 2024-04-23T00:51:33.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.56 MB/sec execute 102 sec latency 89732.998 ms 2024-04-23T00:51:33.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:32.254791+0000 mgr.smithi153.olhmgd (mgr.24421) 104 : cluster [DBG] pgmap v63: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:33.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:32.254791+0000 mgr.smithi153.olhmgd (mgr.24421) 104 : cluster [DBG] pgmap v63: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:34.269 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:51:34.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.54 MB/sec execute 103 sec latency 90733.042 ms 2024-04-23T00:51:34.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:34.018270+0000 mds.foofs.smithi153.uunfav (mds.0) 14 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 80.910080 secs 2024-04-23T00:51:34.877 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:34.018270+0000 mds.foofs.smithi153.uunfav (mds.0) 14 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 80.910080 secs 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (8m) 73s ago 9m 14.9M - 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (9m) 73s ago 9m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (8m) 75s ago 8m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (7m) 73s ago 9m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (4m) 73s ago 4m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (4m) 75s ago 4m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (11m) 73s ago 11m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (2m) 75s ago 8m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (11m) 73s ago 11m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (8m) 75s ago 8m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:51:34.900 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (86s) 73s ago 85s 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (80s) 75s ago 80s 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (9m) 73s ago 9m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (8m) 75s ago 8m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (8m) 73s ago 8m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (7m) 73s ago 7m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (7m) 73s ago 7m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (6m) 73s ago 6m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (6m) 75s ago 6m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (6m) 75s ago 6m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (5m) 75s ago 5m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (5m) 75s ago 5m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:51:34.901 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (7m) 73s ago 8m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:51:35.284 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:51:35.285 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:51:35.600 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:51:35.601 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:51:35.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.53 MB/sec execute 104 sec latency 91733.089 ms 2024-04-23T00:51:35.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:34.255203+0000 mgr.smithi153.olhmgd (mgr.24421) 105 : audit [DBG] from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:35.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:34.255799+0000 mgr.smithi153.olhmgd (mgr.24421) 106 : cluster [DBG] pgmap v64: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:35.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:34.577986+0000 mgr.smithi153.olhmgd (mgr.24421) 107 : audit [DBG] from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:35.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:35.282940+0000 mon.smithi121 (mon.0) 786 : audit [DBG] from='client.? 172.21.15.121:0/2170373172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:51:35.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:34.255203+0000 mgr.smithi153.olhmgd (mgr.24421) 105 : audit [DBG] from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:35.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:34.255799+0000 mgr.smithi153.olhmgd (mgr.24421) 106 : cluster [DBG] pgmap v64: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:35.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:34.577986+0000 mgr.smithi153.olhmgd (mgr.24421) 107 : audit [DBG] from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:35.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:35.282940+0000 mon.smithi121 (mon.0) 786 : audit [DBG] from='client.? 172.21.15.121:0/2170373172' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:51:35.974 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:51:35.974 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:51:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:51:35.974 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:51:35.974 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:51:36.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.51 MB/sec execute 105 sec latency 92733.184 ms 2024-04-23T00:51:36.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:34.886961+0000 mgr.smithi153.olhmgd (mgr.24421) 108 : audit [DBG] from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:36.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:35.599116+0000 mgr.smithi153.olhmgd (mgr.24421) 109 : audit [DBG] from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:36.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:35.973051+0000 mon.smithi121 (mon.0) 787 : audit [DBG] from='client.? 172.21.15.121:0/2980963154' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:51:36.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:34.886961+0000 mgr.smithi153.olhmgd (mgr.24421) 108 : audit [DBG] from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:36.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:35.599116+0000 mgr.smithi153.olhmgd (mgr.24421) 109 : audit [DBG] from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:51:36.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:35.973051+0000 mon.smithi121 (mon.0) 787 : audit [DBG] from='client.? 172.21.15.121:0/2980963154' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:51:37.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.50 MB/sec execute 106 sec latency 93733.236 ms 2024-04-23T00:51:37.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:36.256341+0000 mgr.smithi153.olhmgd (mgr.24421) 110 : cluster [DBG] pgmap v65: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:37.878 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:36.256341+0000 mgr.smithi153.olhmgd (mgr.24421) 110 : cluster [DBG] pgmap v65: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:38.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.48 MB/sec execute 107 sec latency 94733.329 ms 2024-04-23T00:51:39.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:38.256971+0000 mgr.smithi153.olhmgd (mgr.24421) 111 : cluster [DBG] pgmap v66: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:39.018368+0000 mds.foofs.smithi153.uunfav (mds.0) 15 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 85.910192 secs 2024-04-23T00:51:39.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:38.256971+0000 mgr.smithi153.olhmgd (mgr.24421) 111 : cluster [DBG] pgmap v66: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:39.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:39.018368+0000 mds.foofs.smithi153.uunfav (mds.0) 15 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 85.910192 secs 2024-04-23T00:51:39.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.47 MB/sec execute 108 sec latency 95733.412 ms 2024-04-23T00:51:40.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.46 MB/sec execute 109 sec latency 96733.513 ms 2024-04-23T00:51:41.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:40.257458+0000 mgr.smithi153.olhmgd (mgr.24421) 112 : cluster [DBG] pgmap v67: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:41.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:40.257458+0000 mgr.smithi153.olhmgd (mgr.24421) 112 : cluster [DBG] pgmap v67: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:41.672 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.44 MB/sec execute 110 sec latency 97733.608 ms 2024-04-23T00:51:42.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.43 MB/sec execute 111 sec latency 98733.699 ms 2024-04-23T00:51:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:42.257844+0000 mgr.smithi153.olhmgd (mgr.24421) 113 : cluster [DBG] pgmap v68: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:43.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:42.396054+0000 mon.smithi153 (mon.1) 127 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:42.257844+0000 mgr.smithi153.olhmgd (mgr.24421) 113 : cluster [DBG] pgmap v68: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:43.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:42.396054+0000 mon.smithi153 (mon.1) 127 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:43.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.42 MB/sec execute 112 sec latency 99733.798 ms 2024-04-23T00:51:44.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:44.018513+0000 mds.foofs.smithi153.uunfav (mds.0) 16 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 90.910303 secs 2024-04-23T00:51:44.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.40 MB/sec execute 113 sec latency 100733.858 ms 2024-04-23T00:51:44.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:44.018513+0000 mds.foofs.smithi153.uunfav (mds.0) 16 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 90.910303 secs 2024-04-23T00:51:45.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:44.258564+0000 mgr.smithi153.olhmgd (mgr.24421) 114 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:45.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.39 MB/sec execute 114 sec latency 101733.917 ms 2024-04-23T00:51:45.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:44.258564+0000 mgr.smithi153.olhmgd (mgr.24421) 114 : cluster [DBG] pgmap v69: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:46.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.38 MB/sec execute 115 sec latency 102733.996 ms 2024-04-23T00:51:47.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:46.259021+0000 mgr.smithi153.olhmgd (mgr.24421) 115 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:47.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.37 MB/sec execute 116 sec latency 103734.075 ms 2024-04-23T00:51:47.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:46.259021+0000 mgr.smithi153.olhmgd (mgr.24421) 115 : cluster [DBG] pgmap v70: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:48.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.36 MB/sec execute 117 sec latency 104734.122 ms 2024-04-23T00:51:49.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:48.259902+0000 mgr.smithi153.olhmgd (mgr.24421) 116 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:49.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:49.018573+0000 mds.foofs.smithi153.uunfav (mds.0) 17 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 95.910396 secs 2024-04-23T00:51:49.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:48.259902+0000 mgr.smithi153.olhmgd (mgr.24421) 116 : cluster [DBG] pgmap v71: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:49.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:49.018573+0000 mds.foofs.smithi153.uunfav (mds.0) 17 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 95.910396 secs 2024-04-23T00:51:49.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.34 MB/sec execute 118 sec latency 105734.181 ms 2024-04-23T00:51:50.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.33 MB/sec execute 119 sec latency 106734.225 ms 2024-04-23T00:51:51.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:50.260464+0000 mgr.smithi153.olhmgd (mgr.24421) 117 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:51.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:50.260464+0000 mgr.smithi153.olhmgd (mgr.24421) 117 : cluster [DBG] pgmap v72: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:51.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.32 MB/sec execute 120 sec latency 107734.332 ms 2024-04-23T00:51:52.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.31 MB/sec execute 121 sec latency 108734.417 ms 2024-04-23T00:51:53.573 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:52.260841+0000 mgr.smithi153.olhmgd (mgr.24421) 118 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:53.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:52.260841+0000 mgr.smithi153.olhmgd (mgr.24421) 118 : cluster [DBG] pgmap v73: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:53.673 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.30 MB/sec execute 122 sec latency 109734.504 ms 2024-04-23T00:51:54.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:54.018666+0000 mds.foofs.smithi153.uunfav (mds.0) 18 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 100.910489 secs 2024-04-23T00:51:54.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.29 MB/sec execute 123 sec latency 110734.574 ms 2024-04-23T00:51:54.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:54.018666+0000 mds.foofs.smithi153.uunfav (mds.0) 18 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 100.910489 secs 2024-04-23T00:51:55.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:54.261640+0000 mgr.smithi153.olhmgd (mgr.24421) 119 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:55.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.28 MB/sec execute 124 sec latency 111734.684 ms 2024-04-23T00:51:55.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:54.261640+0000 mgr.smithi153.olhmgd (mgr.24421) 119 : cluster [DBG] pgmap v74: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:56.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.27 MB/sec execute 125 sec latency 112734.777 ms 2024-04-23T00:51:57.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:56.262194+0000 mgr.smithi153.olhmgd (mgr.24421) 120 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:57.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.26 MB/sec execute 126 sec latency 113734.873 ms 2024-04-23T00:51:57.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:56.262194+0000 mgr.smithi153.olhmgd (mgr.24421) 120 : cluster [DBG] pgmap v75: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:51:58.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:51:57.396560+0000 mon.smithi153 (mon.1) 128 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:58.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.25 MB/sec execute 127 sec latency 114734.989 ms 2024-04-23T00:51:58.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:51:57.396560+0000 mon.smithi153 (mon.1) 128 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:51:59.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:58.262809+0000 mgr.smithi153.olhmgd (mgr.24421) 121 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:59.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:51:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:51:59.018787+0000 mds.foofs.smithi153.uunfav (mds.0) 19 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 105.910597 secs 2024-04-23T00:51:59.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.24 MB/sec execute 128 sec latency 115735.035 ms 2024-04-23T00:51:59.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:58.262809+0000 mgr.smithi153.olhmgd (mgr.24421) 121 : cluster [DBG] pgmap v76: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:51:59.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:51:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:51:59.018787+0000 mds.foofs.smithi153.uunfav (mds.0) 19 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 105.910597 secs 2024-04-23T00:52:00.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.23 MB/sec execute 129 sec latency 116735.119 ms 2024-04-23T00:52:01.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:00.263348+0000 mgr.smithi153.olhmgd (mgr.24421) 122 : cluster [DBG] pgmap v77: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:01.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.22 MB/sec execute 130 sec latency 117735.208 ms 2024-04-23T00:52:01.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:00.263348+0000 mgr.smithi153.olhmgd (mgr.24421) 122 : cluster [DBG] pgmap v77: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:02.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.21 MB/sec execute 131 sec latency 118735.295 ms 2024-04-23T00:52:03.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:02.263807+0000 mgr.smithi153.olhmgd (mgr.24421) 123 : cluster [DBG] pgmap v78: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:03.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.20 MB/sec execute 132 sec latency 119735.390 ms 2024-04-23T00:52:03.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:02.263807+0000 mgr.smithi153.olhmgd (mgr.24421) 123 : cluster [DBG] pgmap v78: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:04.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:04.018848+0000 mds.foofs.smithi153.uunfav (mds.0) 20 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 110.910669 secs 2024-04-23T00:52:04.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.19 MB/sec execute 133 sec latency 120735.436 ms 2024-04-23T00:52:04.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:04.018848+0000 mds.foofs.smithi153.uunfav (mds.0) 20 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 110.910669 secs 2024-04-23T00:52:05.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:04.264672+0000 mgr.smithi153.olhmgd (mgr.24421) 124 : cluster [DBG] pgmap v79: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:05.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.18 MB/sec execute 134 sec latency 121735.478 ms 2024-04-23T00:52:05.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:04.264672+0000 mgr.smithi153.olhmgd (mgr.24421) 124 : cluster [DBG] pgmap v79: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:06.315 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:52:06.674 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.18 MB/sec execute 135 sec latency 122735.518 ms 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (8m) 105s ago 10m 14.9M - 0.20.0 0881eb8f169f b97c1823535b 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (10m) 105s ago 10m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (9m) 107s ago 9m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (8m) 105s ago 9m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (4m) 105s ago 4m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (4m) 107s ago 4m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:52:06.945 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (11m) 105s ago 11m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (2m) 107s ago 9m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (11m) 105s ago 11m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (9m) 107s ago 9m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (118s) 105s ago 117s 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (112s) 107s ago 112s 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (9m) 105s ago 9m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (8m) 107s ago 8m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (8m) 105s ago 8m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (8m) 105s ago 8m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (7m) 105s ago 7m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (7m) 105s ago 7m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (7m) 107s ago 7m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (6m) 107s ago 6m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:52:06.946 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (6m) 107s ago 6m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:52:06.947 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (6m) 107s ago 6m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:52:06.947 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (8m) 105s ago 9m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:52:07.316 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:07.317 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:52:07.318 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:52:07.318 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:52:07.318 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:52:07.318 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:52:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:06.265060+0000 mgr.smithi153.olhmgd (mgr.24421) 125 : cluster [DBG] pgmap v80: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:06.300458+0000 mgr.smithi153.olhmgd (mgr.24421) 126 : audit [DBG] from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:06.617828+0000 mgr.smithi153.olhmgd (mgr.24421) 127 : audit [DBG] from='client.14852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:07.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:07.315191+0000 mon.smithi121 (mon.0) 788 : audit [DBG] from='client.? 172.21.15.121:0/3232479969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:52:07.629 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:52:07.629 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:52:07.630 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:52:07.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.17 MB/sec execute 136 sec latency 123735.569 ms 2024-04-23T00:52:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:06.265060+0000 mgr.smithi153.olhmgd (mgr.24421) 125 : cluster [DBG] pgmap v80: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:06.300458+0000 mgr.smithi153.olhmgd (mgr.24421) 126 : audit [DBG] from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:06.617828+0000 mgr.smithi153.olhmgd (mgr.24421) 127 : audit [DBG] from='client.14852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:07.315191+0000 mon.smithi121 (mon.0) 788 : audit [DBG] from='client.? 172.21.15.121:0/3232479969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:52:08.005 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:52:08.006 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:52:08.006 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:52:08.006 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:52:08.006 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:52:08.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:06.932332+0000 mgr.smithi153.olhmgd (mgr.24421) 128 : audit [DBG] from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:08.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:07.628136+0000 mgr.smithi153.olhmgd (mgr.24421) 129 : audit [DBG] from='client.14864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:08.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:08.004505+0000 mon.smithi121 (mon.0) 789 : audit [DBG] from='client.? 172.21.15.121:0/3875843777' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:52:08.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.16 MB/sec execute 137 sec latency 124735.681 ms 2024-04-23T00:52:08.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:06.932332+0000 mgr.smithi153.olhmgd (mgr.24421) 128 : audit [DBG] from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:08.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:07.628136+0000 mgr.smithi153.olhmgd (mgr.24421) 129 : audit [DBG] from='client.14864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:08.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:08.004505+0000 mon.smithi121 (mon.0) 789 : audit [DBG] from='client.? 172.21.15.121:0/3875843777' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:52:09.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:08.265990+0000 mgr.smithi153.olhmgd (mgr.24421) 130 : cluster [DBG] pgmap v81: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:09.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:09.018950+0000 mds.foofs.smithi153.uunfav (mds.0) 21 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 115.910775 secs 2024-04-23T00:52:09.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.15 MB/sec execute 138 sec latency 125735.775 ms 2024-04-23T00:52:09.906 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:08.265990+0000 mgr.smithi153.olhmgd (mgr.24421) 130 : cluster [DBG] pgmap v81: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:09.906 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:09.018950+0000 mds.foofs.smithi153.uunfav (mds.0) 21 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 115.910775 secs 2024-04-23T00:52:10.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.14 MB/sec execute 139 sec latency 126735.894 ms 2024-04-23T00:52:11.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:10.266556+0000 mgr.smithi153.olhmgd (mgr.24421) 131 : cluster [DBG] pgmap v82: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:11.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.13 MB/sec execute 140 sec latency 127735.973 ms 2024-04-23T00:52:11.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:10.266556+0000 mgr.smithi153.olhmgd (mgr.24421) 131 : cluster [DBG] pgmap v82: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:12.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.13 MB/sec execute 141 sec latency 128736.054 ms 2024-04-23T00:52:13.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:12.266925+0000 mgr.smithi153.olhmgd (mgr.24421) 132 : cluster [DBG] pgmap v83: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:13.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:12.396860+0000 mon.smithi153 (mon.1) 129 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:13.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.12 MB/sec execute 142 sec latency 129736.152 ms 2024-04-23T00:52:13.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:12.266925+0000 mgr.smithi153.olhmgd (mgr.24421) 132 : cluster [DBG] pgmap v83: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:13.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:12.396860+0000 mon.smithi153 (mon.1) 129 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:14.019049+0000 mds.foofs.smithi153.uunfav (mds.0) 22 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 120.910843 secs 2024-04-23T00:52:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:14.019058+0000 mds.foofs.smithi153.uunfav (mds.0) 23 : cluster [WRN] slow request 120.910843 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:52:14.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:14.019097+0000 mds.foofs.smithi153.uunfav (mds.0) 24 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 120.910895 seconds ago 2024-04-23T00:52:14.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.11 MB/sec execute 143 sec latency 130736.225 ms 2024-04-23T00:52:14.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:14.019049+0000 mds.foofs.smithi153.uunfav (mds.0) 22 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 120.910843 secs 2024-04-23T00:52:14.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:14.019058+0000 mds.foofs.smithi153.uunfav (mds.0) 23 : cluster [WRN] slow request 120.910843 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:52:14.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:14.019097+0000 mds.foofs.smithi153.uunfav (mds.0) 24 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 120.910895 seconds ago 2024-04-23T00:52:15.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:14.267733+0000 mgr.smithi153.olhmgd (mgr.24421) 133 : cluster [DBG] pgmap v84: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:15.675 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.10 MB/sec execute 144 sec latency 131736.305 ms 2024-04-23T00:52:15.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:14.267733+0000 mgr.smithi153.olhmgd (mgr.24421) 133 : cluster [DBG] pgmap v84: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:16.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.09 MB/sec execute 145 sec latency 132736.402 ms 2024-04-23T00:52:17.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:16.268112+0000 mgr.smithi153.olhmgd (mgr.24421) 134 : cluster [DBG] pgmap v85: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:17.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.09 MB/sec execute 146 sec latency 133736.531 ms 2024-04-23T00:52:17.823 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:16.268112+0000 mgr.smithi153.olhmgd (mgr.24421) 134 : cluster [DBG] pgmap v85: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:18.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.08 MB/sec execute 147 sec latency 134736.571 ms 2024-04-23T00:52:19.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:18.268817+0000 mgr.smithi153.olhmgd (mgr.24421) 135 : cluster [DBG] pgmap v86: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:19.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:19.019172+0000 mds.foofs.smithi153.uunfav (mds.0) 25 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 125.910982 secs 2024-04-23T00:52:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:18.268817+0000 mgr.smithi153.olhmgd (mgr.24421) 135 : cluster [DBG] pgmap v86: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:19.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:19.019172+0000 mds.foofs.smithi153.uunfav (mds.0) 25 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 125.910982 secs 2024-04-23T00:52:19.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.07 MB/sec execute 148 sec latency 135736.660 ms 2024-04-23T00:52:20.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.06 MB/sec execute 149 sec latency 136736.766 ms 2024-04-23T00:52:21.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:20.269325+0000 mgr.smithi153.olhmgd (mgr.24421) 136 : cluster [DBG] pgmap v87: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:21.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.06 MB/sec execute 150 sec latency 137736.846 ms 2024-04-23T00:52:21.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:20.269325+0000 mgr.smithi153.olhmgd (mgr.24421) 136 : cluster [DBG] pgmap v87: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:22.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.05 MB/sec execute 151 sec latency 138736.922 ms 2024-04-23T00:52:23.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:22.269902+0000 mgr.smithi153.olhmgd (mgr.24421) 137 : cluster [DBG] pgmap v88: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:23.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.04 MB/sec execute 152 sec latency 139737.002 ms 2024-04-23T00:52:23.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:22.269902+0000 mgr.smithi153.olhmgd (mgr.24421) 137 : cluster [DBG] pgmap v88: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:24.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:24.019322+0000 mds.foofs.smithi153.uunfav (mds.0) 26 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 130.911126 secs 2024-04-23T00:52:24.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.04 MB/sec execute 153 sec latency 140737.092 ms 2024-04-23T00:52:24.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:24.019322+0000 mds.foofs.smithi153.uunfav (mds.0) 26 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 130.911126 secs 2024-04-23T00:52:25.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:24.270628+0000 mgr.smithi153.olhmgd (mgr.24421) 138 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:25.676 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.03 MB/sec execute 154 sec latency 141737.173 ms 2024-04-23T00:52:25.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:24.270628+0000 mgr.smithi153.olhmgd (mgr.24421) 138 : cluster [DBG] pgmap v89: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:26.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.02 MB/sec execute 155 sec latency 142737.257 ms 2024-04-23T00:52:27.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:26.271206+0000 mgr.smithi153.olhmgd (mgr.24421) 139 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:27.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.02 MB/sec execute 156 sec latency 143737.348 ms 2024-04-23T00:52:27.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:26.271206+0000 mgr.smithi153.olhmgd (mgr.24421) 139 : cluster [DBG] pgmap v90: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:28.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:27.397155+0000 mon.smithi153 (mon.1) 130 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:28.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.01 MB/sec execute 157 sec latency 144737.397 ms 2024-04-23T00:52:28.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:27.397155+0000 mon.smithi153 (mon.1) 130 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:29.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:28.271938+0000 mgr.smithi153.olhmgd (mgr.24421) 140 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:29.378 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:29.019436+0000 mds.foofs.smithi153.uunfav (mds.0) 27 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 135.911241 secs 2024-04-23T00:52:29.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.00 MB/sec execute 158 sec latency 145737.517 ms 2024-04-23T00:52:29.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:28.271938+0000 mgr.smithi153.olhmgd (mgr.24421) 140 : cluster [DBG] pgmap v91: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:29.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:29.019436+0000 mds.foofs.smithi153.uunfav (mds.0) 27 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 135.911241 secs 2024-04-23T00:52:30.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 1.00 MB/sec execute 159 sec latency 146737.579 ms 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:30.272307+0000 mgr.smithi153.olhmgd (mgr.24421) 141 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.276712+0000 mon.smithi121 (mon.0) 790 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.282836+0000 mon.smithi121 (mon.0) 791 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:52:30.331030+0000 mgr.smithi153.olhmgd (mgr.24421) 142 : cephadm [INF] Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:52:30.339258+0000 mgr.smithi153.olhmgd (mgr.24421) 143 : cephadm [INF] Regenerating cephadm self-signed grafana TLS certificates 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.395298+0000 mon.smithi121 (mon.0) 792 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.399178+0000 mon.smithi121 (mon.0) 793 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.402402+0000 mon.smithi153 (mon.1) 131 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:30.402793+0000 mgr.smithi153.olhmgd (mgr.24421) 144 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:52:31.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:52:30.404610+0000 mgr.smithi153.olhmgd (mgr.24421) 145 : cephadm [INF] Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T00:52:31.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:30.272307+0000 mgr.smithi153.olhmgd (mgr.24421) 141 : cluster [DBG] pgmap v92: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:31.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.276712+0000 mon.smithi121 (mon.0) 790 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.282836+0000 mon.smithi121 (mon.0) 791 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:52:30.331030+0000 mgr.smithi153.olhmgd (mgr.24421) 142 : cephadm [INF] Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:52:30.339258+0000 mgr.smithi153.olhmgd (mgr.24421) 143 : cephadm [INF] Regenerating cephadm self-signed grafana TLS certificates 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.395298+0000 mon.smithi121 (mon.0) 792 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.399178+0000 mon.smithi121 (mon.0) 793 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.402402+0000 mon.smithi153 (mon.1) 131 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:30.402793+0000 mgr.smithi153.olhmgd (mgr.24421) 144 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-23T00:52:31.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:52:30.404610+0000 mgr.smithi153.olhmgd (mgr.24421) 145 : cephadm [INF] Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T00:52:31.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.99 MB/sec execute 160 sec latency 147737.663 ms 2024-04-23T00:52:32.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.99 MB/sec execute 161 sec latency 148737.726 ms 2024-04-23T00:52:33.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:32.272820+0000 mgr.smithi153.olhmgd (mgr.24421) 146 : cluster [DBG] pgmap v93: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:33.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:32.272820+0000 mgr.smithi153.olhmgd (mgr.24421) 146 : cluster [DBG] pgmap v93: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:33.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.98 MB/sec execute 162 sec latency 149737.801 ms 2024-04-23T00:52:34.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:34.019574+0000 mds.foofs.smithi153.uunfav (mds.0) 28 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 140.911381 secs 2024-04-23T00:52:34.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:34.019574+0000 mds.foofs.smithi153.uunfav (mds.0) 28 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 140.911381 secs 2024-04-23T00:52:34.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.97 MB/sec execute 163 sec latency 150737.861 ms 2024-04-23T00:52:35.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:34.273711+0000 mgr.smithi153.olhmgd (mgr.24421) 147 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:35.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:34.273711+0000 mgr.smithi153.olhmgd (mgr.24421) 147 : cluster [DBG] pgmap v94: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:35.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.97 MB/sec execute 164 sec latency 151737.887 ms 2024-04-23T00:52:36.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.96 MB/sec execute 165 sec latency 152737.937 ms 2024-04-23T00:52:37.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:36.274230+0000 mgr.smithi153.olhmgd (mgr.24421) 148 : cluster [DBG] pgmap v95: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:37.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:36.274230+0000 mgr.smithi153.olhmgd (mgr.24421) 148 : cluster [DBG] pgmap v95: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:37.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.96 MB/sec execute 166 sec latency 153737.966 ms 2024-04-23T00:52:38.379 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:52:38.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.95 MB/sec execute 167 sec latency 154737.998 ms 2024-04-23T00:52:39.106 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:52:39.106 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (10m) 2m ago 10m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (9m) 2m ago 9m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (8m) 2m ago 10m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (5m) 2m ago 5m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (5m) 2m ago 5m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (12m) 2m ago 12m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (3m) 2m ago 9m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (12m) 2m ago 12m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (9m) 2m ago 9m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (2m) 2m ago 2m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (2m) 2m ago 2m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (10m) 2m ago 10m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (9m) 2m ago 9m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:52:39.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (9m) 2m ago 9m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (8m) 2m ago 8m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (8m) 2m ago 8m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (7m) 2m ago 7m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (7m) 2m ago 7m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (7m) 2m ago 7m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (7m) 2m ago 6m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (6m) 2m ago 6m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:52:39.108 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (8m) 2m ago 9m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:52:39.551 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:52:39.551 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:52:39.551 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:52:39.551 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:39.551 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:52:39.552 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:52:39.553 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:52:39.553 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:52:39.553 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:52:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:38.274981+0000 mgr.smithi153.olhmgd (mgr.24421) 149 : cluster [DBG] pgmap v96: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:38.356777+0000 mgr.smithi153.olhmgd (mgr.24421) 150 : audit [DBG] from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:38.745719+0000 mgr.smithi153.olhmgd (mgr.24421) 151 : audit [DBG] from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:39.019671+0000 mds.foofs.smithi153.uunfav (mds.0) 29 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 145.911491 secs 2024-04-23T00:52:39.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:38.274981+0000 mgr.smithi153.olhmgd (mgr.24421) 149 : cluster [DBG] pgmap v96: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:39.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:38.356777+0000 mgr.smithi153.olhmgd (mgr.24421) 150 : audit [DBG] from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:39.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:38.745719+0000 mgr.smithi153.olhmgd (mgr.24421) 151 : audit [DBG] from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:39.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:39.019671+0000 mds.foofs.smithi153.uunfav (mds.0) 29 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 145.911491 secs 2024-04-23T00:52:39.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.94 MB/sec execute 168 sec latency 155738.026 ms 2024-04-23T00:52:39.936 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:52:39.936 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:52:39.936 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:52:39.936 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:52:39.936 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:52:39.937 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:52:39.937 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:52:39.937 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:52:39.937 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:52:40.360 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:52:40.361 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:52:40.361 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:52:40.361 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:52:40.361 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:52:40.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:39.094056+0000 mgr.smithi153.olhmgd (mgr.24421) 152 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:40.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:39.549052+0000 mon.smithi153 (mon.1) 132 : audit [DBG] from='client.? 172.21.15.121:0/1361603061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:52:40.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:39.094056+0000 mgr.smithi153.olhmgd (mgr.24421) 152 : audit [DBG] from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:40.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:39.549052+0000 mon.smithi153 (mon.1) 132 : audit [DBG] from='client.? 172.21.15.121:0/1361603061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:52:40.677 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.94 MB/sec execute 169 sec latency 156738.055 ms 2024-04-23T00:52:41.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:39.933716+0000 mgr.smithi153.olhmgd (mgr.24421) 153 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:41.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:40.275370+0000 mgr.smithi153.olhmgd (mgr.24421) 154 : cluster [DBG] pgmap v97: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:41.530 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:40.359062+0000 mon.smithi121 (mon.0) 794 : audit [DBG] from='client.? 172.21.15.121:0/391281987' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:52:41.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:39.933716+0000 mgr.smithi153.olhmgd (mgr.24421) 153 : audit [DBG] from='client.14886 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:52:41.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:40.275370+0000 mgr.smithi153.olhmgd (mgr.24421) 154 : cluster [DBG] pgmap v97: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:41.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:40.359062+0000 mon.smithi121 (mon.0) 794 : audit [DBG] from='client.? 172.21.15.121:0/391281987' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:52:41.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.93 MB/sec execute 170 sec latency 157738.130 ms 2024-04-23T00:52:42.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.93 MB/sec execute 171 sec latency 158738.164 ms 2024-04-23T00:52:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:42.275732+0000 mgr.smithi153.olhmgd (mgr.24421) 155 : cluster [DBG] pgmap v98: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:42.397434+0000 mon.smithi153 (mon.1) 133 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:43.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.92 MB/sec execute 172 sec latency 159738.713 ms 2024-04-23T00:52:43.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:42.275732+0000 mgr.smithi153.olhmgd (mgr.24421) 155 : cluster [DBG] pgmap v98: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:43.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:42.397434+0000 mon.smithi153 (mon.1) 133 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:44.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:44.019828+0000 mds.foofs.smithi153.uunfav (mds.0) 30 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 150.911625 secs 2024-04-23T00:52:44.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.92 MB/sec execute 173 sec latency 160738.753 ms 2024-04-23T00:52:44.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:44.019828+0000 mds.foofs.smithi153.uunfav (mds.0) 30 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 150.911625 secs 2024-04-23T00:52:45.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:44.276423+0000 mgr.smithi153.olhmgd (mgr.24421) 156 : cluster [DBG] pgmap v99: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:45.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.91 MB/sec execute 174 sec latency 161738.854 ms 2024-04-23T00:52:45.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:44.276423+0000 mgr.smithi153.olhmgd (mgr.24421) 156 : cluster [DBG] pgmap v99: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:46.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.91 MB/sec execute 175 sec latency 162738.955 ms 2024-04-23T00:52:47.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:46.277006+0000 mgr.smithi153.olhmgd (mgr.24421) 157 : cluster [DBG] pgmap v100: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:47.678 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.90 MB/sec execute 176 sec latency 163739.046 ms 2024-04-23T00:52:47.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:46.277006+0000 mgr.smithi153.olhmgd (mgr.24421) 157 : cluster [DBG] pgmap v100: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:48.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.90 MB/sec execute 177 sec latency 164739.117 ms 2024-04-23T00:52:49.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:48.277894+0000 mgr.smithi153.olhmgd (mgr.24421) 158 : cluster [DBG] pgmap v101: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:49.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:49.019912+0000 mds.foofs.smithi153.uunfav (mds.0) 31 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 155.911732 secs 2024-04-23T00:52:49.628 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:48.277894+0000 mgr.smithi153.olhmgd (mgr.24421) 158 : cluster [DBG] pgmap v101: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:49.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:49.019912+0000 mds.foofs.smithi153.uunfav (mds.0) 31 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 155.911732 secs 2024-04-23T00:52:49.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.89 MB/sec execute 178 sec latency 165739.199 ms 2024-04-23T00:52:50.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.89 MB/sec execute 179 sec latency 166739.290 ms 2024-04-23T00:52:51.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:50.278417+0000 mgr.smithi153.olhmgd (mgr.24421) 159 : cluster [DBG] pgmap v102: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:51.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.88 MB/sec execute 180 sec latency 167739.385 ms 2024-04-23T00:52:51.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:50.278417+0000 mgr.smithi153.olhmgd (mgr.24421) 159 : cluster [DBG] pgmap v102: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:52.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.88 MB/sec execute 181 sec latency 168739.482 ms 2024-04-23T00:52:53.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:52.279084+0000 mgr.smithi153.olhmgd (mgr.24421) 160 : cluster [DBG] pgmap v103: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:53.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.87 MB/sec execute 182 sec latency 169739.571 ms 2024-04-23T00:52:53.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:52.279084+0000 mgr.smithi153.olhmgd (mgr.24421) 160 : cluster [DBG] pgmap v103: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:54.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:54.019993+0000 mds.foofs.smithi153.uunfav (mds.0) 32 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 160.911815 secs 2024-04-23T00:52:54.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.87 MB/sec execute 183 sec latency 170739.647 ms 2024-04-23T00:52:54.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:54.019993+0000 mds.foofs.smithi153.uunfav (mds.0) 32 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 160.911815 secs 2024-04-23T00:52:55.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:54.279776+0000 mgr.smithi153.olhmgd (mgr.24421) 161 : cluster [DBG] pgmap v104: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:55.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.86 MB/sec execute 184 sec latency 171739.742 ms 2024-04-23T00:52:55.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:54.279776+0000 mgr.smithi153.olhmgd (mgr.24421) 161 : cluster [DBG] pgmap v104: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:56.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.86 MB/sec execute 185 sec latency 172739.847 ms 2024-04-23T00:52:57.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:56.280174+0000 mgr.smithi153.olhmgd (mgr.24421) 162 : cluster [DBG] pgmap v105: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:57.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.85 MB/sec execute 186 sec latency 173739.953 ms 2024-04-23T00:52:57.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:56.280174+0000 mgr.smithi153.olhmgd (mgr.24421) 162 : cluster [DBG] pgmap v105: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:52:58.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:52:57.397791+0000 mon.smithi153 (mon.1) 134 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:58.679 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.85 MB/sec execute 187 sec latency 174740.010 ms 2024-04-23T00:52:58.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:52:57.397791+0000 mon.smithi153 (mon.1) 134 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:52:59.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:58.280990+0000 mgr.smithi153.olhmgd (mgr.24421) 163 : cluster [DBG] pgmap v106: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:59.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:52:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:52:59.020143+0000 mds.foofs.smithi153.uunfav (mds.0) 33 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 165.911948 secs 2024-04-23T00:52:59.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.84 MB/sec execute 188 sec latency 175740.052 ms 2024-04-23T00:52:59.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:58.280990+0000 mgr.smithi153.olhmgd (mgr.24421) 163 : cluster [DBG] pgmap v106: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:52:59.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:52:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:52:59.020143+0000 mds.foofs.smithi153.uunfav (mds.0) 33 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 165.911948 secs 2024-04-23T00:53:00.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.84 MB/sec execute 189 sec latency 176740.175 ms 2024-04-23T00:53:01.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:00.281527+0000 mgr.smithi153.olhmgd (mgr.24421) 164 : cluster [DBG] pgmap v107: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:01.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.83 MB/sec execute 190 sec latency 177740.278 ms 2024-04-23T00:53:01.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:00.281527+0000 mgr.smithi153.olhmgd (mgr.24421) 164 : cluster [DBG] pgmap v107: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:02.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.83 MB/sec execute 191 sec latency 178740.360 ms 2024-04-23T00:53:03.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:02.282082+0000 mgr.smithi153.olhmgd (mgr.24421) 165 : cluster [DBG] pgmap v108: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:03.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.83 MB/sec execute 192 sec latency 179740.453 ms 2024-04-23T00:53:03.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:02.282082+0000 mgr.smithi153.olhmgd (mgr.24421) 165 : cluster [DBG] pgmap v108: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:04.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:04.020246+0000 mds.foofs.smithi153.uunfav (mds.0) 34 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 170.912064 secs 2024-04-23T00:53:04.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.82 MB/sec execute 193 sec latency 180740.571 ms 2024-04-23T00:53:04.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:04.020246+0000 mds.foofs.smithi153.uunfav (mds.0) 34 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 170.912064 secs 2024-04-23T00:53:05.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:04.282991+0000 mgr.smithi153.olhmgd (mgr.24421) 166 : cluster [DBG] pgmap v109: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:05.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.82 MB/sec execute 194 sec latency 181740.657 ms 2024-04-23T00:53:05.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:04.282991+0000 mgr.smithi153.olhmgd (mgr.24421) 166 : cluster [DBG] pgmap v109: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:06.680 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.81 MB/sec execute 195 sec latency 182740.769 ms 2024-04-23T00:53:07.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:06.283551+0000 mgr.smithi153.olhmgd (mgr.24421) 167 : cluster [DBG] pgmap v110: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:07.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.81 MB/sec execute 196 sec latency 183740.871 ms 2024-04-23T00:53:07.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:06.283551+0000 mgr.smithi153.olhmgd (mgr.24421) 167 : cluster [DBG] pgmap v110: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:08.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.81 MB/sec execute 197 sec latency 184740.934 ms 2024-04-23T00:53:09.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:08.284410+0000 mgr.smithi153.olhmgd (mgr.24421) 168 : cluster [DBG] pgmap v111: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:09.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:09.020393+0000 mds.foofs.smithi153.uunfav (mds.0) 35 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 175.912198 secs 2024-04-23T00:53:09.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:08.284410+0000 mgr.smithi153.olhmgd (mgr.24421) 168 : cluster [DBG] pgmap v111: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:09.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:09.020393+0000 mds.foofs.smithi153.uunfav (mds.0) 35 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 175.912198 secs 2024-04-23T00:53:09.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.80 MB/sec execute 198 sec latency 185741.047 ms 2024-04-23T00:53:10.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.80 MB/sec execute 199 sec latency 186741.098 ms 2024-04-23T00:53:10.703 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:53:11.322 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:53:11.322 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:53:11.322 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (11m) 2m ago 11m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (10m) 2m ago 10m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (9m) 2m ago 10m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (5m) 2m ago 5m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (5m) 2m ago 5m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (12m) 2m ago 12m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (3m) 2m ago 10m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (12m) 2m ago 12m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (10m) 2m ago 10m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (3m) 2m ago 3m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (2m) 2m ago 2m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (10m) 2m ago 10m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (9m) 2m ago 9m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (9m) 2m ago 9m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:53:11.323 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (9m) 2m ago 9m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (8m) 2m ago 8m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (8m) 2m ago 8m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (8m) 2m ago 8m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (7m) 2m ago 7m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (7m) 2m ago 7m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (7m) 2m ago 7m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:53:11.324 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (9m) 2m ago 10m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:53:11.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:10.284875+0000 mgr.smithi153.olhmgd (mgr.24421) 169 : cluster [DBG] pgmap v112: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:11.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:10.688668+0000 mgr.smithi153.olhmgd (mgr.24421) 170 : audit [DBG] from='client.14894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:11.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.79 MB/sec execute 200 sec latency 187741.146 ms 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:53:11.694 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:53:11.695 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:53:11.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:10.284875+0000 mgr.smithi153.olhmgd (mgr.24421) 169 : cluster [DBG] pgmap v112: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:11.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:10.688668+0000 mgr.smithi153.olhmgd (mgr.24421) 170 : audit [DBG] from='client.14894 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:53:12.010 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:53:12.387 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:53:12.387 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:53:12.387 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:53:12.387 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:53:12.387 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:53:12.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:11.004040+0000 mgr.smithi153.olhmgd (mgr.24421) 171 : audit [DBG] from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:12.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:11.309119+0000 mgr.smithi153.olhmgd (mgr.24421) 172 : audit [DBG] from='client.14902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:12.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:11.692049+0000 mon.smithi121 (mon.0) 795 : audit [DBG] from='client.? 172.21.15.121:0/802730534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:53:12.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:11.004040+0000 mgr.smithi153.olhmgd (mgr.24421) 171 : audit [DBG] from='client.14898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:12.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:11.309119+0000 mgr.smithi153.olhmgd (mgr.24421) 172 : audit [DBG] from='client.14902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:12.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:11.692049+0000 mon.smithi121 (mon.0) 795 : audit [DBG] from='client.? 172.21.15.121:0/802730534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:53:12.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.79 MB/sec execute 201 sec latency 188741.220 ms 2024-04-23T00:53:13.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:12.007627+0000 mgr.smithi153.olhmgd (mgr.24421) 173 : audit [DBG] from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:13.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:12.285324+0000 mgr.smithi153.olhmgd (mgr.24421) 174 : cluster [DBG] pgmap v113: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:13.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:12.385157+0000 mon.smithi121 (mon.0) 796 : audit [DBG] from='client.? 172.21.15.121:0/2962744852' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:53:13.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:12.398213+0000 mon.smithi153 (mon.1) 135 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:13.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.79 MB/sec execute 202 sec latency 189741.302 ms 2024-04-23T00:53:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:12.007627+0000 mgr.smithi153.olhmgd (mgr.24421) 173 : audit [DBG] from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:12.285324+0000 mgr.smithi153.olhmgd (mgr.24421) 174 : cluster [DBG] pgmap v113: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:12.385157+0000 mon.smithi121 (mon.0) 796 : audit [DBG] from='client.? 172.21.15.121:0/2962744852' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:53:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:12.398213+0000 mon.smithi153 (mon.1) 135 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:14.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:14.020452+0000 mds.foofs.smithi153.uunfav (mds.0) 36 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 180.912257 secs 2024-04-23T00:53:14.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.78 MB/sec execute 203 sec latency 190741.415 ms 2024-04-23T00:53:14.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:14.020452+0000 mds.foofs.smithi153.uunfav (mds.0) 36 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 180.912257 secs 2024-04-23T00:53:15.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:14.286076+0000 mgr.smithi153.olhmgd (mgr.24421) 175 : cluster [DBG] pgmap v114: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:15.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.78 MB/sec execute 204 sec latency 191741.504 ms 2024-04-23T00:53:15.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:14.286076+0000 mgr.smithi153.olhmgd (mgr.24421) 175 : cluster [DBG] pgmap v114: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:16.681 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.77 MB/sec execute 205 sec latency 192741.603 ms 2024-04-23T00:53:17.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:16.286653+0000 mgr.smithi153.olhmgd (mgr.24421) 176 : cluster [DBG] pgmap v115: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:17.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.77 MB/sec execute 206 sec latency 193741.712 ms 2024-04-23T00:53:17.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:16.286653+0000 mgr.smithi153.olhmgd (mgr.24421) 176 : cluster [DBG] pgmap v115: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:18.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.77 MB/sec execute 207 sec latency 194741.809 ms 2024-04-23T00:53:19.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:18.287401+0000 mgr.smithi153.olhmgd (mgr.24421) 177 : cluster [DBG] pgmap v116: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:19.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:19.020557+0000 mds.foofs.smithi153.uunfav (mds.0) 37 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 185.912376 secs 2024-04-23T00:53:19.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:18.287401+0000 mgr.smithi153.olhmgd (mgr.24421) 177 : cluster [DBG] pgmap v116: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:19.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:19.020557+0000 mds.foofs.smithi153.uunfav (mds.0) 37 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 185.912376 secs 2024-04-23T00:53:19.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.76 MB/sec execute 208 sec latency 195741.934 ms 2024-04-23T00:53:20.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.76 MB/sec execute 209 sec latency 196742.007 ms 2024-04-23T00:53:21.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:20.287804+0000 mgr.smithi153.olhmgd (mgr.24421) 178 : cluster [DBG] pgmap v117: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:21.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.76 MB/sec execute 210 sec latency 197742.107 ms 2024-04-23T00:53:21.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:20.287804+0000 mgr.smithi153.olhmgd (mgr.24421) 178 : cluster [DBG] pgmap v117: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:22.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.75 MB/sec execute 211 sec latency 198742.166 ms 2024-04-23T00:53:23.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:22.288255+0000 mgr.smithi153.olhmgd (mgr.24421) 179 : cluster [DBG] pgmap v118: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:23.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.75 MB/sec execute 212 sec latency 199742.214 ms 2024-04-23T00:53:23.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:22.288255+0000 mgr.smithi153.olhmgd (mgr.24421) 179 : cluster [DBG] pgmap v118: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:24.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:24.020685+0000 mds.foofs.smithi153.uunfav (mds.0) 38 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 190.912489 secs 2024-04-23T00:53:24.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.74 MB/sec execute 213 sec latency 200742.314 ms 2024-04-23T00:53:24.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:24.020685+0000 mds.foofs.smithi153.uunfav (mds.0) 38 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 190.912489 secs 2024-04-23T00:53:25.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:24.289169+0000 mgr.smithi153.olhmgd (mgr.24421) 180 : cluster [DBG] pgmap v119: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:25.682 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.74 MB/sec execute 214 sec latency 201742.410 ms 2024-04-23T00:53:25.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:24.289169+0000 mgr.smithi153.olhmgd (mgr.24421) 180 : cluster [DBG] pgmap v119: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:26.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.74 MB/sec execute 215 sec latency 202742.489 ms 2024-04-23T00:53:27.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:26.289720+0000 mgr.smithi153.olhmgd (mgr.24421) 181 : cluster [DBG] pgmap v120: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:27.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.73 MB/sec execute 216 sec latency 203742.584 ms 2024-04-23T00:53:27.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:26.289720+0000 mgr.smithi153.olhmgd (mgr.24421) 181 : cluster [DBG] pgmap v120: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:28.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:27.398560+0000 mon.smithi153 (mon.1) 136 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:28.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.73 MB/sec execute 217 sec latency 204742.705 ms 2024-04-23T00:53:28.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:27.398560+0000 mon.smithi153 (mon.1) 136 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:29.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:28.290327+0000 mgr.smithi153.olhmgd (mgr.24421) 182 : cluster [DBG] pgmap v121: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:29.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:29.020814+0000 mds.foofs.smithi153.uunfav (mds.0) 39 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 195.912619 secs 2024-04-23T00:53:29.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.73 MB/sec execute 218 sec latency 205742.818 ms 2024-04-23T00:53:29.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:28.290327+0000 mgr.smithi153.olhmgd (mgr.24421) 182 : cluster [DBG] pgmap v121: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:29.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:29.020814+0000 mds.foofs.smithi153.uunfav (mds.0) 39 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 195.912619 secs 2024-04-23T00:53:30.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.72 MB/sec execute 219 sec latency 206742.920 ms 2024-04-23T00:53:31.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:30.290767+0000 mgr.smithi153.olhmgd (mgr.24421) 183 : cluster [DBG] pgmap v122: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:31.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.72 MB/sec execute 220 sec latency 207743.019 ms 2024-04-23T00:53:31.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:30.290767+0000 mgr.smithi153.olhmgd (mgr.24421) 183 : cluster [DBG] pgmap v122: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:32.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.72 MB/sec execute 221 sec latency 208743.113 ms 2024-04-23T00:53:33.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:32.291190+0000 mgr.smithi153.olhmgd (mgr.24421) 184 : cluster [DBG] pgmap v123: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:33.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.71 MB/sec execute 222 sec latency 209743.208 ms 2024-04-23T00:53:33.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:32.291190+0000 mgr.smithi153.olhmgd (mgr.24421) 184 : cluster [DBG] pgmap v123: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:34.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:34.020939+0000 mds.foofs.smithi153.uunfav (mds.0) 40 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 200.912746 secs 2024-04-23T00:53:34.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.71 MB/sec execute 223 sec latency 210743.310 ms 2024-04-23T00:53:34.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:34.020939+0000 mds.foofs.smithi153.uunfav (mds.0) 40 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 200.912746 secs 2024-04-23T00:53:35.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:34.292113+0000 mgr.smithi153.olhmgd (mgr.24421) 185 : cluster [DBG] pgmap v124: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:35.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.71 MB/sec execute 224 sec latency 211743.402 ms 2024-04-23T00:53:35.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:34.292113+0000 mgr.smithi153.olhmgd (mgr.24421) 185 : cluster [DBG] pgmap v124: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:36.683 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.71 MB/sec execute 225 sec latency 212743.483 ms 2024-04-23T00:53:37.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.70 MB/sec execute 226 sec latency 213743.591 ms 2024-04-23T00:53:37.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:36.292579+0000 mgr.smithi153.olhmgd (mgr.24421) 186 : cluster [DBG] pgmap v125: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:37.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:36.292579+0000 mgr.smithi153.olhmgd (mgr.24421) 186 : cluster [DBG] pgmap v125: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:38.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.70 MB/sec execute 227 sec latency 214743.675 ms 2024-04-23T00:53:39.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:38.293407+0000 mgr.smithi153.olhmgd (mgr.24421) 187 : cluster [DBG] pgmap v126: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:39.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:39.021013+0000 mds.foofs.smithi153.uunfav (mds.0) 41 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 205.912817 secs 2024-04-23T00:53:39.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:38.293407+0000 mgr.smithi153.olhmgd (mgr.24421) 187 : cluster [DBG] pgmap v126: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:39.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:39.021013+0000 mds.foofs.smithi153.uunfav (mds.0) 41 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 205.912817 secs 2024-04-23T00:53:39.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.70 MB/sec execute 228 sec latency 215743.776 ms 2024-04-23T00:53:40.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.69 MB/sec execute 229 sec latency 216743.861 ms 2024-04-23T00:53:41.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:40.293880+0000 mgr.smithi153.olhmgd (mgr.24421) 188 : cluster [DBG] pgmap v127: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:41.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.69 MB/sec execute 230 sec latency 217743.935 ms 2024-04-23T00:53:41.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:40.293880+0000 mgr.smithi153.olhmgd (mgr.24421) 188 : cluster [DBG] pgmap v127: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:42.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.69 MB/sec execute 231 sec latency 218744.001 ms 2024-04-23T00:53:42.721 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:53:43.339 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:53:43.339 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:53:43.339 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (11m) 3m ago 11m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:53:43.339 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (10m) 3m ago 10m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (10m) 3m ago 11m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (6m) 3m ago 6m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (6m) 3m ago 6m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (13m) 3m ago 13m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (4m) 3m ago 10m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (13m) 3m ago 13m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (10m) 3m ago 10m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (3m) 3m ago 3m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (3m) 3m ago 3m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (11m) 3m ago 11m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (10m) 3m ago 10m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (10m) 3m ago 10m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (9m) 3m ago 9m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (9m) 3m ago 9m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:53:43.340 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (9m) 3m ago 9m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:53:43.341 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (8m) 3m ago 8m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:53:43.341 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (8m) 3m ago 8m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:53:43.341 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (8m) 3m ago 8m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:53:43.341 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (7m) 3m ago 7m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:53:43.341 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (9m) 3m ago 11m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:53:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:42.294329+0000 mgr.smithi153.olhmgd (mgr.24421) 189 : cluster [DBG] pgmap v128: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:42.398769+0000 mon.smithi153 (mon.1) 137 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:43.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:42.706158+0000 mgr.smithi153.olhmgd (mgr.24421) 190 : audit [DBG] from='client.14918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:43.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.68 MB/sec execute 232 sec latency 219744.052 ms 2024-04-23T00:53:43.711 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:53:43.712 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:53:43.713 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:53:43.713 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:53:43.713 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:53:43.713 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:53:43.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:42.294329+0000 mgr.smithi153.olhmgd (mgr.24421) 189 : cluster [DBG] pgmap v128: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:43.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:42.398769+0000 mon.smithi153 (mon.1) 137 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:43.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:42.706158+0000 mgr.smithi153.olhmgd (mgr.24421) 190 : audit [DBG] from='client.14918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:44.026 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:53:44.026 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:53:44.027 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:53:44.400 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:53:44.400 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:53:44.400 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:53:44.400 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:53:44.400 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:53:44.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:43.019991+0000 mgr.smithi153.olhmgd (mgr.24421) 191 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:44.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:43.326520+0000 mgr.smithi153.olhmgd (mgr.24421) 192 : audit [DBG] from='client.14926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:44.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:43.709524+0000 mon.smithi121 (mon.0) 797 : audit [DBG] from='client.? 172.21.15.121:0/4139421183' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:53:44.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:44.021103+0000 mds.foofs.smithi153.uunfav (mds.0) 42 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 210.912906 secs 2024-04-23T00:53:44.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.68 MB/sec execute 233 sec latency 220744.079 ms 2024-04-23T00:53:44.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:43.019991+0000 mgr.smithi153.olhmgd (mgr.24421) 191 : audit [DBG] from='client.14922 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:44.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:43.326520+0000 mgr.smithi153.olhmgd (mgr.24421) 192 : audit [DBG] from='client.14926 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:44.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:43.709524+0000 mon.smithi121 (mon.0) 797 : audit [DBG] from='client.? 172.21.15.121:0/4139421183' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:53:44.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:44.021103+0000 mds.foofs.smithi153.uunfav (mds.0) 42 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 210.912906 secs 2024-04-23T00:53:45.612 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:44.023709+0000 mgr.smithi153.olhmgd (mgr.24421) 193 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:45.612 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:44.294982+0000 mgr.smithi153.olhmgd (mgr.24421) 194 : cluster [DBG] pgmap v129: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:45.612 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:44.398072+0000 mon.smithi121 (mon.0) 798 : audit [DBG] from='client.? 172.21.15.121:0/2133750670' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:53:45.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.68 MB/sec execute 234 sec latency 221744.104 ms 2024-04-23T00:53:45.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:44.023709+0000 mgr.smithi153.olhmgd (mgr.24421) 193 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:53:45.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:44.294982+0000 mgr.smithi153.olhmgd (mgr.24421) 194 : cluster [DBG] pgmap v129: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:45.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:44.398072+0000 mon.smithi121 (mon.0) 798 : audit [DBG] from='client.? 172.21.15.121:0/2133750670' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:53:46.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.68 MB/sec execute 235 sec latency 222744.164 ms 2024-04-23T00:53:47.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:46.295487+0000 mgr.smithi153.olhmgd (mgr.24421) 195 : cluster [DBG] pgmap v130: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:47.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.67 MB/sec execute 236 sec latency 223744.249 ms 2024-04-23T00:53:47.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:46.295487+0000 mgr.smithi153.olhmgd (mgr.24421) 195 : cluster [DBG] pgmap v130: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:48.684 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.67 MB/sec execute 237 sec latency 224744.328 ms 2024-04-23T00:53:49.574 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:48.296172+0000 mgr.smithi153.olhmgd (mgr.24421) 196 : cluster [DBG] pgmap v131: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:49.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:49.021218+0000 mds.foofs.smithi153.uunfav (mds.0) 43 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 215.913022 secs 2024-04-23T00:53:49.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:48.296172+0000 mgr.smithi153.olhmgd (mgr.24421) 196 : cluster [DBG] pgmap v131: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:49.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:49.021218+0000 mds.foofs.smithi153.uunfav (mds.0) 43 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 215.913022 secs 2024-04-23T00:53:49.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.67 MB/sec execute 238 sec latency 225744.416 ms 2024-04-23T00:53:50.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.66 MB/sec execute 239 sec latency 226744.516 ms 2024-04-23T00:53:51.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:50.296579+0000 mgr.smithi153.olhmgd (mgr.24421) 197 : cluster [DBG] pgmap v132: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:51.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.66 MB/sec execute 240 sec latency 227744.627 ms 2024-04-23T00:53:51.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:50.296579+0000 mgr.smithi153.olhmgd (mgr.24421) 197 : cluster [DBG] pgmap v132: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:52.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.66 MB/sec execute 241 sec latency 228744.747 ms 2024-04-23T00:53:53.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:52.297041+0000 mgr.smithi153.olhmgd (mgr.24421) 198 : cluster [DBG] pgmap v133: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:53.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.66 MB/sec execute 242 sec latency 229744.825 ms 2024-04-23T00:53:53.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:52.297041+0000 mgr.smithi153.olhmgd (mgr.24421) 198 : cluster [DBG] pgmap v133: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:54.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:54.021267+0000 mds.foofs.smithi153.uunfav (mds.0) 44 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 220.913089 secs 2024-04-23T00:53:54.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.65 MB/sec execute 243 sec latency 230744.911 ms 2024-04-23T00:53:54.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:54.021267+0000 mds.foofs.smithi153.uunfav (mds.0) 44 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 220.913089 secs 2024-04-23T00:53:55.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:54.297795+0000 mgr.smithi153.olhmgd (mgr.24421) 199 : cluster [DBG] pgmap v134: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:55.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.65 MB/sec execute 244 sec latency 231744.992 ms 2024-04-23T00:53:55.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:54.297795+0000 mgr.smithi153.olhmgd (mgr.24421) 199 : cluster [DBG] pgmap v134: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:56.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.65 MB/sec execute 245 sec latency 232745.088 ms 2024-04-23T00:53:57.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:56.298243+0000 mgr.smithi153.olhmgd (mgr.24421) 200 : cluster [DBG] pgmap v135: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:57.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.64 MB/sec execute 246 sec latency 233745.191 ms 2024-04-23T00:53:57.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:56.298243+0000 mgr.smithi153.olhmgd (mgr.24421) 200 : cluster [DBG] pgmap v135: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:53:58.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:53:57.399286+0000 mon.smithi153 (mon.1) 138 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:58.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.64 MB/sec execute 247 sec latency 234745.271 ms 2024-04-23T00:53:58.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:53:57.399286+0000 mon.smithi153 (mon.1) 138 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:53:59.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:58.299023+0000 mgr.smithi153.olhmgd (mgr.24421) 201 : cluster [DBG] pgmap v136: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:59.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:53:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:53:59.021422+0000 mds.foofs.smithi153.uunfav (mds.0) 45 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 225.913226 secs 2024-04-23T00:53:59.685 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.64 MB/sec execute 248 sec latency 235745.394 ms 2024-04-23T00:53:59.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:58.299023+0000 mgr.smithi153.olhmgd (mgr.24421) 201 : cluster [DBG] pgmap v136: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:53:59.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:53:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:53:59.021422+0000 mds.foofs.smithi153.uunfav (mds.0) 45 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 225.913226 secs 2024-04-23T00:54:00.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.64 MB/sec execute 249 sec latency 236745.494 ms 2024-04-23T00:54:01.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:00.299548+0000 mgr.smithi153.olhmgd (mgr.24421) 202 : cluster [DBG] pgmap v137: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:01.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.63 MB/sec execute 250 sec latency 237745.528 ms 2024-04-23T00:54:01.824 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:00.299548+0000 mgr.smithi153.olhmgd (mgr.24421) 202 : cluster [DBG] pgmap v137: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:02.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.63 MB/sec execute 251 sec latency 238745.627 ms 2024-04-23T00:54:03.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:02.300049+0000 mgr.smithi153.olhmgd (mgr.24421) 203 : cluster [DBG] pgmap v138: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:03.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.63 MB/sec execute 252 sec latency 239745.752 ms 2024-04-23T00:54:03.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:02.300049+0000 mgr.smithi153.olhmgd (mgr.24421) 203 : cluster [DBG] pgmap v138: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:04.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.63 MB/sec execute 253 sec latency 240745.856 ms 2024-04-23T00:54:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:04.021537+0000 mds.foofs.smithi153.uunfav (mds.0) 46 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 230.913356 secs 2024-04-23T00:54:04.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:04.021537+0000 mds.foofs.smithi153.uunfav (mds.0) 46 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 230.913356 secs 2024-04-23T00:54:05.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.62 MB/sec execute 254 sec latency 241745.944 ms 2024-04-23T00:54:05.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:04.300869+0000 mgr.smithi153.olhmgd (mgr.24421) 204 : cluster [DBG] pgmap v139: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:05.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:04.300869+0000 mgr.smithi153.olhmgd (mgr.24421) 204 : cluster [DBG] pgmap v139: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:06.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.62 MB/sec execute 255 sec latency 242746.024 ms 2024-04-23T00:54:07.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.62 MB/sec execute 256 sec latency 243746.108 ms 2024-04-23T00:54:07.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:06.301291+0000 mgr.smithi153.olhmgd (mgr.24421) 205 : cluster [DBG] pgmap v140: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:07.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:06.301291+0000 mgr.smithi153.olhmgd (mgr.24421) 205 : cluster [DBG] pgmap v140: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:08.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.62 MB/sec execute 257 sec latency 244746.200 ms 2024-04-23T00:54:09.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:08.302062+0000 mgr.smithi153.olhmgd (mgr.24421) 206 : cluster [DBG] pgmap v141: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:09.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:09.021664+0000 mds.foofs.smithi153.uunfav (mds.0) 47 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 235.913469 secs 2024-04-23T00:54:09.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:08.302062+0000 mgr.smithi153.olhmgd (mgr.24421) 206 : cluster [DBG] pgmap v141: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:09.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:09.021664+0000 mds.foofs.smithi153.uunfav (mds.0) 47 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 235.913469 secs 2024-04-23T00:54:09.686 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.61 MB/sec execute 258 sec latency 245746.297 ms 2024-04-23T00:54:10.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.61 MB/sec execute 259 sec latency 246746.392 ms 2024-04-23T00:54:11.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:10.302513+0000 mgr.smithi153.olhmgd (mgr.24421) 207 : cluster [DBG] pgmap v142: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:11.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.61 MB/sec execute 260 sec latency 247746.451 ms 2024-04-23T00:54:11.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:10.302513+0000 mgr.smithi153.olhmgd (mgr.24421) 207 : cluster [DBG] pgmap v142: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:12.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.61 MB/sec execute 261 sec latency 248746.598 ms 2024-04-23T00:54:13.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:12.302918+0000 mgr.smithi153.olhmgd (mgr.24421) 208 : cluster [DBG] pgmap v143: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:13.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:12.399449+0000 mon.smithi153 (mon.1) 139 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:13.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.61 MB/sec execute 262 sec latency 249746.663 ms 2024-04-23T00:54:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:12.302918+0000 mgr.smithi153.olhmgd (mgr.24421) 208 : cluster [DBG] pgmap v143: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:12.399449+0000 mon.smithi153 (mon.1) 139 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:14.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:14.021779+0000 mds.foofs.smithi153.uunfav (mds.0) 48 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 240.913571 secs 2024-04-23T00:54:14.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:14.021787+0000 mds.foofs.smithi153.uunfav (mds.0) 49 : cluster [WRN] slow request 240.913570 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:54:14.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:14 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:14.021831+0000 mds.foofs.smithi153.uunfav (mds.0) 50 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 240.913629 seconds ago 2024-04-23T00:54:14.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.60 MB/sec execute 263 sec latency 250746.692 ms 2024-04-23T00:54:14.735 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:54:14.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:14.021779+0000 mds.foofs.smithi153.uunfav (mds.0) 48 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 240.913571 secs 2024-04-23T00:54:14.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:14.021787+0000 mds.foofs.smithi153.uunfav (mds.0) 49 : cluster [WRN] slow request 240.913570 seconds old, received at 2024-04-23T00:50:13.108156+0000: client_request(client.14760:4 getattr pAsLsXsFs #0x10000004a2d 2024-04-23T00:50:13.108081+0000 caller_uid=0, caller_gid=0{0,}) currently failed to rdlock, waiting 2024-04-23T00:54:14.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:14.021831+0000 mds.foofs.smithi153.uunfav (mds.0) 50 : cluster [WRN] client.14568 isn't responding to mclientcaps(revoke), ino 0x10000004a2d pending pAsxLsXsxFcb issued pAsxLsXsxFsxcrwb, sent 240.913629 seconds ago 2024-04-23T00:54:15.358 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (12m) 3m ago 12m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (11m) 3m ago 11m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (10m) 3m ago 12m 32.7M - 6.7.4 557c83e11646 6903f605b995 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (6m) 3m ago 6m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (6m) 3m ago 6m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (13m) 3m ago 13m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (4m) 3m ago 11m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (13m) 3m ago 13m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (11m) 3m ago 11m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (4m) 3m ago 4m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (4m) 3m ago 4m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:54:15.359 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (11m) 3m ago 11m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (11m) 3m ago 11m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (10m) 3m ago 10m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (10m) 3m ago 10m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (9m) 3m ago 9m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (9m) 3m ago 9m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (9m) 3m ago 9m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (8m) 3m ago 8m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (8m) 3m ago 8m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (8m) 3m ago 8m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:54:15.360 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (10m) 3m ago 11m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:54:15.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:14.303695+0000 mgr.smithi153.olhmgd (mgr.24421) 209 : cluster [DBG] pgmap v144: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:15.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:14.720691+0000 mgr.smithi153.olhmgd (mgr.24421) 210 : audit [DBG] from='client.14942 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:15.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.60 MB/sec execute 264 sec latency 251746.734 ms 2024-04-23T00:54:15.734 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:54:15.735 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:54:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:54:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:54:15.736 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:54:15.736 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:54:15.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:14.303695+0000 mgr.smithi153.olhmgd (mgr.24421) 209 : cluster [DBG] pgmap v144: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:15.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:14.720691+0000 mgr.smithi153.olhmgd (mgr.24421) 210 : audit [DBG] from='client.14942 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:54:16.048 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:54:16.049 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:54:16.427 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:54:16.427 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:54:16.427 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:54:16.427 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:54:16.427 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:54:16.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:15.036693+0000 mgr.smithi153.olhmgd (mgr.24421) 211 : audit [DBG] from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:16.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:15.345216+0000 mgr.smithi153.olhmgd (mgr.24421) 212 : audit [DBG] from='client.14950 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:16.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:15.732299+0000 mon.smithi121 (mon.0) 799 : audit [DBG] from='client.? 172.21.15.121:0/4274875305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:54:16.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.60 MB/sec execute 265 sec latency 252746.801 ms 2024-04-23T00:54:16.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:15.036693+0000 mgr.smithi153.olhmgd (mgr.24421) 211 : audit [DBG] from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:16.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:15.345216+0000 mgr.smithi153.olhmgd (mgr.24421) 212 : audit [DBG] from='client.14950 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:16.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:15.732299+0000 mon.smithi121 (mon.0) 799 : audit [DBG] from='client.? 172.21.15.121:0/4274875305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:54:17.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:16.045097+0000 mgr.smithi153.olhmgd (mgr.24421) 213 : audit [DBG] from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:17.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:16.304168+0000 mgr.smithi153.olhmgd (mgr.24421) 214 : cluster [DBG] pgmap v145: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:17.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:16.424587+0000 mon.smithi121 (mon.0) 800 : audit [DBG] from='client.? 172.21.15.121:0/3946604224' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:54:17.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.60 MB/sec execute 266 sec latency 253746.910 ms 2024-04-23T00:54:17.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:16.045097+0000 mgr.smithi153.olhmgd (mgr.24421) 213 : audit [DBG] from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:17.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:16.304168+0000 mgr.smithi153.olhmgd (mgr.24421) 214 : cluster [DBG] pgmap v145: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:17.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:16.424587+0000 mon.smithi121 (mon.0) 800 : audit [DBG] from='client.? 172.21.15.121:0/3946604224' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:54:18.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.59 MB/sec execute 267 sec latency 254746.981 ms 2024-04-23T00:54:19.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:18.304955+0000 mgr.smithi153.olhmgd (mgr.24421) 215 : cluster [DBG] pgmap v146: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:19.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:19.021882+0000 mds.foofs.smithi153.uunfav (mds.0) 51 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 245.913682 secs 2024-04-23T00:54:19.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:18.304955+0000 mgr.smithi153.olhmgd (mgr.24421) 215 : cluster [DBG] pgmap v146: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:19.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:19.021882+0000 mds.foofs.smithi153.uunfav (mds.0) 51 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 245.913682 secs 2024-04-23T00:54:19.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.59 MB/sec execute 268 sec latency 255747.080 ms 2024-04-23T00:54:20.687 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.59 MB/sec execute 269 sec latency 256747.172 ms 2024-04-23T00:54:21.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:20.305348+0000 mgr.smithi153.olhmgd (mgr.24421) 216 : cluster [DBG] pgmap v147: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:21.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.59 MB/sec execute 270 sec latency 257747.239 ms 2024-04-23T00:54:21.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:20.305348+0000 mgr.smithi153.olhmgd (mgr.24421) 216 : cluster [DBG] pgmap v147: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:22.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.59 MB/sec execute 271 sec latency 258747.344 ms 2024-04-23T00:54:23.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:22.305885+0000 mgr.smithi153.olhmgd (mgr.24421) 217 : cluster [DBG] pgmap v148: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:23.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.58 MB/sec execute 272 sec latency 259747.439 ms 2024-04-23T00:54:23.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:22.305885+0000 mgr.smithi153.olhmgd (mgr.24421) 217 : cluster [DBG] pgmap v148: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:24.629 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:24.022014+0000 mds.foofs.smithi153.uunfav (mds.0) 52 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 250.913817 secs 2024-04-23T00:54:24.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.58 MB/sec execute 273 sec latency 260747.532 ms 2024-04-23T00:54:24.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:24.022014+0000 mds.foofs.smithi153.uunfav (mds.0) 52 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 250.913817 secs 2024-04-23T00:54:25.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:24.306690+0000 mgr.smithi153.olhmgd (mgr.24421) 218 : cluster [DBG] pgmap v149: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:25.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.58 MB/sec execute 274 sec latency 261747.615 ms 2024-04-23T00:54:25.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:24.306690+0000 mgr.smithi153.olhmgd (mgr.24421) 218 : cluster [DBG] pgmap v149: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:26.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.58 MB/sec execute 275 sec latency 262747.726 ms 2024-04-23T00:54:27.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:26.307293+0000 mgr.smithi153.olhmgd (mgr.24421) 219 : cluster [DBG] pgmap v150: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:27.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.57 MB/sec execute 276 sec latency 263747.816 ms 2024-04-23T00:54:27.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:26.307293+0000 mgr.smithi153.olhmgd (mgr.24421) 219 : cluster [DBG] pgmap v150: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:28.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.57 MB/sec execute 277 sec latency 264747.910 ms 2024-04-23T00:54:28.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:27.399961+0000 mon.smithi153 (mon.1) 140 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:28.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:27.399961+0000 mon.smithi153 (mon.1) 140 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:29.688 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.57 MB/sec execute 278 sec latency 265747.980 ms 2024-04-23T00:54:29.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:28.308068+0000 mgr.smithi153.olhmgd (mgr.24421) 220 : cluster [DBG] pgmap v151: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:29.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:29.022118+0000 mds.foofs.smithi153.uunfav (mds.0) 53 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 255.913922 secs 2024-04-23T00:54:29.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:28.308068+0000 mgr.smithi153.olhmgd (mgr.24421) 220 : cluster [DBG] pgmap v151: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:29.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:29.022118+0000 mds.foofs.smithi153.uunfav (mds.0) 53 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 255.913922 secs 2024-04-23T00:54:30.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.57 MB/sec execute 279 sec latency 266748.090 ms 2024-04-23T00:54:31.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.57 MB/sec execute 280 sec latency 267748.224 ms 2024-04-23T00:54:31.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:30.308637+0000 mgr.smithi153.olhmgd (mgr.24421) 221 : cluster [DBG] pgmap v152: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:31.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:30.308637+0000 mgr.smithi153.olhmgd (mgr.24421) 221 : cluster [DBG] pgmap v152: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:32.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.56 MB/sec execute 281 sec latency 268748.321 ms 2024-04-23T00:54:33.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.56 MB/sec execute 282 sec latency 269748.436 ms 2024-04-23T00:54:33.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:32.308979+0000 mgr.smithi153.olhmgd (mgr.24421) 222 : cluster [DBG] pgmap v153: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:33.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:32.308979+0000 mgr.smithi153.olhmgd (mgr.24421) 222 : cluster [DBG] pgmap v153: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:34.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.56 MB/sec execute 283 sec latency 270748.537 ms 2024-04-23T00:54:34.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:34.022229+0000 mds.foofs.smithi153.uunfav (mds.0) 54 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 260.914051 secs 2024-04-23T00:54:34.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:34.022229+0000 mds.foofs.smithi153.uunfav (mds.0) 54 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 260.914051 secs 2024-04-23T00:54:35.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.56 MB/sec execute 284 sec latency 271748.584 ms 2024-04-23T00:54:35.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:34.309808+0000 mgr.smithi153.olhmgd (mgr.24421) 223 : cluster [DBG] pgmap v154: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:35.879 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:34.309808+0000 mgr.smithi153.olhmgd (mgr.24421) 223 : cluster [DBG] pgmap v154: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:36.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.56 MB/sec execute 285 sec latency 272748.673 ms 2024-04-23T00:54:37.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 286 sec latency 273748.789 ms 2024-04-23T00:54:37.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:36.310233+0000 mgr.smithi153.olhmgd (mgr.24421) 224 : cluster [DBG] pgmap v155: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:37.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:36.310233+0000 mgr.smithi153.olhmgd (mgr.24421) 224 : cluster [DBG] pgmap v155: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:38.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 287 sec latency 274748.875 ms 2024-04-23T00:54:39.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:38.310876+0000 mgr.smithi153.olhmgd (mgr.24421) 225 : cluster [DBG] pgmap v156: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:39.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:39.022316+0000 mds.foofs.smithi153.uunfav (mds.0) 55 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 265.914122 secs 2024-04-23T00:54:39.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:38.310876+0000 mgr.smithi153.olhmgd (mgr.24421) 225 : cluster [DBG] pgmap v156: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:39.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:39.022316+0000 mds.foofs.smithi153.uunfav (mds.0) 55 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 265.914122 secs 2024-04-23T00:54:39.689 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 288 sec latency 275748.963 ms 2024-04-23T00:54:40.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 289 sec latency 276749.041 ms 2024-04-23T00:54:41.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:40.311350+0000 mgr.smithi153.olhmgd (mgr.24421) 226 : cluster [DBG] pgmap v157: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:41.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 290 sec latency 277749.127 ms 2024-04-23T00:54:41.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:40.311350+0000 mgr.smithi153.olhmgd (mgr.24421) 226 : cluster [DBG] pgmap v157: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:42.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.55 MB/sec execute 291 sec latency 278749.213 ms 2024-04-23T00:54:43.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:42.311935+0000 mgr.smithi153.olhmgd (mgr.24421) 227 : cluster [DBG] pgmap v158: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:43.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:42.400036+0000 mon.smithi153 (mon.1) 141 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:43.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.54 MB/sec execute 292 sec latency 279749.318 ms 2024-04-23T00:54:43.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:42.311935+0000 mgr.smithi153.olhmgd (mgr.24421) 227 : cluster [DBG] pgmap v158: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:43.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:42.400036+0000 mon.smithi153 (mon.1) 141 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:44.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:44.022383+0000 mds.foofs.smithi153.uunfav (mds.0) 56 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 270.914187 secs 2024-04-23T00:54:44.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.54 MB/sec execute 293 sec latency 280749.417 ms 2024-04-23T00:54:44.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:44.022383+0000 mds.foofs.smithi153.uunfav (mds.0) 56 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 270.914187 secs 2024-04-23T00:54:45.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:44.312703+0000 mgr.smithi153.olhmgd (mgr.24421) 228 : cluster [DBG] pgmap v159: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:45.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.54 MB/sec execute 294 sec latency 281749.510 ms 2024-04-23T00:54:45.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:44.312703+0000 mgr.smithi153.olhmgd (mgr.24421) 228 : cluster [DBG] pgmap v159: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:46.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.54 MB/sec execute 295 sec latency 282749.579 ms 2024-04-23T00:54:46.777 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (12m) 4m ago 12m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (11m) 4m ago 11m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:54:47.434 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (7m) 4m ago 7m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (7m) 4m ago 7m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (14m) 4m ago 14m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (5m) 4m ago 11m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (14m) 4m ago 14m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (11m) 4m ago 11m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (4m) 4m ago 4m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (4m) 4m ago 4m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (12m) 4m ago 12m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (11m) 4m ago 11m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (11m) 4m ago 11m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (10m) 4m ago 10m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (10m) 4m ago 10m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (10m) 4m ago 10m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:54:47.435 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (9m) 4m ago 9m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:54:47.436 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (9m) 4m ago 9m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:54:47.436 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (9m) 4m ago 9m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:54:47.436 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (8m) 4m ago 8m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:54:47.436 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (10m) 4m ago 12m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:54:47.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.54 MB/sec execute 296 sec latency 283749.628 ms 2024-04-23T00:54:47.806 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:54:47.806 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:54:47.806 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:54:47.806 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:47.806 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:54:47.807 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:54:48.075 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:46.313226+0000 mgr.smithi153.olhmgd (mgr.24421) 229 : cluster [DBG] pgmap v160: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:48.075 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:46.761000+0000 mgr.smithi153.olhmgd (mgr.24421) 230 : audit [DBG] from='client.14966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:48.075 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:46.821444+0000 mon.smithi121 (mon.0) 801 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:54:48.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:46.828769+0000 mon.smithi121 (mon.0) 802 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:54:48.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:54:46.850989+0000 mgr.smithi153.olhmgd (mgr.24421) 231 : cephadm [INF] Reconfiguring node-exporter.smithi121 (dependencies changed)... 2024-04-23T00:54:48.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:54:46.851383+0000 mgr.smithi153.olhmgd (mgr.24421) 232 : cephadm [INF] Reconfiguring daemon node-exporter.smithi121 on smithi121 2024-04-23T00:54:48.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:47.803462+0000 mon.smithi121 (mon.0) 803 : audit [DBG] from='client.? 172.21.15.121:0/4178651625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:54:48.123 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:54:48.129 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:46.313226+0000 mgr.smithi153.olhmgd (mgr.24421) 229 : cluster [DBG] pgmap v160: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:46.761000+0000 mgr.smithi153.olhmgd (mgr.24421) 230 : audit [DBG] from='client.14966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:46.821444+0000 mon.smithi121 (mon.0) 801 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:46.828769+0000 mon.smithi121 (mon.0) 802 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:54:46.850989+0000 mgr.smithi153.olhmgd (mgr.24421) 231 : cephadm [INF] Reconfiguring node-exporter.smithi121 (dependencies changed)... 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:54:46.851383+0000 mgr.smithi153.olhmgd (mgr.24421) 232 : cephadm [INF] Reconfiguring daemon node-exporter.smithi121 on smithi121 2024-04-23T00:54:48.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:47.803462+0000 mon.smithi121 (mon.0) 803 : audit [DBG] from='client.? 172.21.15.121:0/4178651625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:54:48.496 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests 2024-04-23T00:54:48.496 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release 2024-04-23T00:54:48.496 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): Client smithi121:foo1 failing to respond to capability release client_id: 14568 2024-04-23T00:54:48.497 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests 2024-04-23T00:54:48.497 INFO:teuthology.orchestra.run.smithi121.stdout: mds.foofs.smithi153.uunfav(mds.0): 1 slow requests are blocked > 30 secs 2024-04-23T00:54:48.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 297 sec latency 284749.750 ms 2024-04-23T00:54:49.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:47.110104+0000 mgr.smithi153.olhmgd (mgr.24421) 233 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:49.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:47.420215+0000 mgr.smithi153.olhmgd (mgr.24421) 234 : audit [DBG] from='client.14974 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:49.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:48.493798+0000 mon.smithi121 (mon.0) 804 : audit [DBG] from='client.? 172.21.15.121:0/3002612401' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:54:49.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:47.110104+0000 mgr.smithi153.olhmgd (mgr.24421) 233 : audit [DBG] from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:49.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:47.420215+0000 mgr.smithi153.olhmgd (mgr.24421) 234 : audit [DBG] from='client.14974 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:49.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:48.493798+0000 mon.smithi121 (mon.0) 804 : audit [DBG] from='client.? 172.21.15.121:0/3002612401' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:54:49.690 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 298 sec latency 285749.839 ms 2024-04-23T00:54:50.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:48.119567+0000 mgr.smithi153.olhmgd (mgr.24421) 235 : audit [DBG] from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:50.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:48.314008+0000 mgr.smithi153.olhmgd (mgr.24421) 236 : cluster [DBG] pgmap v161: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:50.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:49.022465+0000 mds.foofs.smithi153.uunfav (mds.0) 57 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 275.914283 secs 2024-04-23T00:54:50.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:49.022530+0000 mds.foofs.smithi153.uunfav (mds.0) 58 : cluster [WRN] evicting unresponsive client smithi153:foo1 (24299), after 302.304 seconds 2024-04-23T00:54:50.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:49.022539+0000 mds.foofs.smithi153.uunfav (mds.0) 59 : cluster [INF] Evicting (and blocklisting) client session 24299 (172.21.15.153:0/2214368580) 2024-04-23T00:54:50.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:49.022821+0000 mon.smithi153 (mon.1) 142 : audit [INF] from='mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855]' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]: dispatch 2024-04-23T00:54:50.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:49.023430+0000 mon.smithi121 (mon.0) 805 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]: dispatch 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:48.119567+0000 mgr.smithi153.olhmgd (mgr.24421) 235 : audit [DBG] from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:48.314008+0000 mgr.smithi153.olhmgd (mgr.24421) 236 : cluster [DBG] pgmap v161: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:49.022465+0000 mds.foofs.smithi153.uunfav (mds.0) 57 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 275.914283 secs 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:49.022530+0000 mds.foofs.smithi153.uunfav (mds.0) 58 : cluster [WRN] evicting unresponsive client smithi153:foo1 (24299), after 302.304 seconds 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:49.022539+0000 mds.foofs.smithi153.uunfav (mds.0) 59 : cluster [INF] Evicting (and blocklisting) client session 24299 (172.21.15.153:0/2214368580) 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:49.022821+0000 mon.smithi153 (mon.1) 142 : audit [INF] from='mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855]' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]: dispatch 2024-04-23T00:54:50.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:49.023430+0000 mon.smithi121 (mon.0) 805 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]: dispatch 2024-04-23T00:54:50.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 299 sec latency 286749.901 ms 2024-04-23T00:54:51.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:49.827610+0000 mon.smithi121 (mon.0) 806 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]': finished 2024-04-23T00:54:51.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:49.827702+0000 mon.smithi121 (mon.0) 807 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-04-23T00:54:51.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:49.827610+0000 mon.smithi121 (mon.0) 806 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/2214368580"}]': finished 2024-04-23T00:54:51.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:49.827702+0000 mon.smithi121 (mon.0) 807 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-04-23T00:54:51.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 300 sec latency 287750.000 ms 2024-04-23T00:54:52.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:50.314617+0000 mgr.smithi153.olhmgd (mgr.24421) 237 : cluster [DBG] pgmap v163: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:54:52.261 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:50.314617+0000 mgr.smithi153.olhmgd (mgr.24421) 237 : cluster [DBG] pgmap v163: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:54:52.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 301 sec latency 288750.096 ms 2024-04-23T00:54:53.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.53 MB/sec execute 302 sec latency 289750.210 ms 2024-04-23T00:54:54.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:52.315098+0000 mgr.smithi153.olhmgd (mgr.24421) 238 : cluster [DBG] pgmap v164: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:54:54.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:52.315098+0000 mgr.smithi153.olhmgd (mgr.24421) 238 : cluster [DBG] pgmap v164: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:54:54.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 303 sec latency 290750.325 ms 2024-04-23T00:54:55.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:54.022595+0000 mds.foofs.smithi153.uunfav (mds.0) 60 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 280.914397 secs 2024-04-23T00:54:55.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:54.022595+0000 mds.foofs.smithi153.uunfav (mds.0) 60 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 280.914397 secs 2024-04-23T00:54:55.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 304 sec latency 291750.414 ms 2024-04-23T00:54:56.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:54.315953+0000 mgr.smithi153.olhmgd (mgr.24421) 239 : cluster [DBG] pgmap v165: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:56.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:54.315953+0000 mgr.smithi153.olhmgd (mgr.24421) 239 : cluster [DBG] pgmap v165: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:56.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 305 sec latency 292750.497 ms 2024-04-23T00:54:57.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 306 sec latency 293750.592 ms 2024-04-23T00:54:58.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:56.316546+0000 mgr.smithi153.olhmgd (mgr.24421) 240 : cluster [DBG] pgmap v166: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:58.130 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:54:57.400408+0000 mon.smithi153 (mon.1) 143 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:58.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:56.316546+0000 mgr.smithi153.olhmgd (mgr.24421) 240 : cluster [DBG] pgmap v166: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:58.325 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:54:57.400408+0000 mon.smithi153 (mon.1) 143 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:54:58.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 307 sec latency 294750.673 ms 2024-04-23T00:54:59.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:58.317156+0000 mgr.smithi153.olhmgd (mgr.24421) 241 : cluster [DBG] pgmap v167: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:59.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:54:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:54:59.022666+0000 mds.foofs.smithi153.uunfav (mds.0) 61 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 285.914471 secs 2024-04-23T00:54:59.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:58.317156+0000 mgr.smithi153.olhmgd (mgr.24421) 241 : cluster [DBG] pgmap v167: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 204 B/s wr, 0 op/s 2024-04-23T00:54:59.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:54:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:54:59.022666+0000 mds.foofs.smithi153.uunfav (mds.0) 61 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 285.914471 secs 2024-04-23T00:54:59.691 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.52 MB/sec execute 308 sec latency 295750.758 ms 2024-04-23T00:55:00.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.51 MB/sec execute 309 sec latency 296750.804 ms 2024-04-23T00:55:01.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:00.317599+0000 mgr.smithi153.olhmgd (mgr.24421) 242 : cluster [DBG] pgmap v168: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 195 B/s wr, 0 op/s 2024-04-23T00:55:01.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.51 MB/sec execute 310 sec latency 297750.915 ms 2024-04-23T00:55:01.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:00.317599+0000 mgr.smithi153.olhmgd (mgr.24421) 242 : cluster [DBG] pgmap v168: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 195 B/s wr, 0 op/s 2024-04-23T00:55:02.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.51 MB/sec execute 311 sec latency 298751.038 ms 2024-04-23T00:55:03.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:02.317998+0000 mgr.smithi153.olhmgd (mgr.24421) 243 : cluster [DBG] pgmap v169: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-23T00:55:03.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104898 0.51 MB/sec execute 312 sec latency 299751.140 ms 2024-04-23T00:55:03.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:02.317998+0000 mgr.smithi153.olhmgd (mgr.24421) 243 : cluster [DBG] pgmap v169: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-23T00:55:04.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 104966 0.52 MB/sec execute 313 sec latency 300665.129 ms 2024-04-23T00:55:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:04.022745+0000 mds.foofs.smithi153.uunfav (mds.0) 62 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 290.914547 secs 2024-04-23T00:55:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:04.022792+0000 mds.foofs.smithi153.uunfav (mds.0) 63 : cluster [WRN] evicting unresponsive client smithi121:foo1 (14568), after 315.195 seconds 2024-04-23T00:55:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:04.022802+0000 mds.foofs.smithi153.uunfav (mds.0) 64 : cluster [INF] Evicting (and blocklisting) client session 14568 (172.21.15.121:0/1071333478) 2024-04-23T00:55:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:04.023082+0000 mon.smithi153 (mon.1) 144 : audit [INF] from='mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855]' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]: dispatch 2024-04-23T00:55:04.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:04.023500+0000 mon.smithi121 (mon.0) 808 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]: dispatch 2024-04-23T00:55:04.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:04.022745+0000 mds.foofs.smithi153.uunfav (mds.0) 62 : cluster [WRN] 1 slow requests, 0 included below; oldest blocked for > 290.914547 secs 2024-04-23T00:55:04.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:04.022792+0000 mds.foofs.smithi153.uunfav (mds.0) 63 : cluster [WRN] evicting unresponsive client smithi121:foo1 (14568), after 315.195 seconds 2024-04-23T00:55:04.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:04.022802+0000 mds.foofs.smithi153.uunfav (mds.0) 64 : cluster [INF] Evicting (and blocklisting) client session 14568 (172.21.15.121:0/1071333478) 2024-04-23T00:55:04.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:04.023082+0000 mon.smithi153 (mon.1) 144 : audit [INF] from='mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855]' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]: dispatch 2024-04-23T00:55:04.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:04.023500+0000 mon.smithi121 (mon.0) 808 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]: dispatch 2024-04-23T00:55:05.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 105452 0.56 MB/sec execute 314 sec latency 34.955 ms 2024-04-23T00:55:05.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:04.318758+0000 mgr.smithi153.olhmgd (mgr.24421) 244 : cluster [DBG] pgmap v170: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-23T00:55:05.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:04.380495+0000 mon.smithi121 (mon.0) 809 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]': finished 2024-04-23T00:55:05.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:04.380599+0000 mon.smithi121 (mon.0) 810 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-04-23T00:55:05.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:04.318758+0000 mgr.smithi153.olhmgd (mgr.24421) 244 : cluster [DBG] pgmap v170: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-23T00:55:05.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:04.380495+0000 mon.smithi121 (mon.0) 809 : audit [INF] from='mds.? ' entity='mds.foofs.smithi153.uunfav' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/1071333478"}]': finished 2024-04-23T00:55:05.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:04.380599+0000 mon.smithi121 (mon.0) 810 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-04-23T00:55:06.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 105941 0.62 MB/sec execute 315 sec latency 29.775 ms 2024-04-23T00:55:07.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 106391 0.67 MB/sec execute 316 sec latency 37.031 ms 2024-04-23T00:55:07.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:06.319122+0000 mgr.smithi153.olhmgd (mgr.24421) 245 : cluster [DBG] pgmap v172: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:55:07.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:06.319122+0000 mgr.smithi153.olhmgd (mgr.24421) 245 : cluster [DBG] pgmap v172: 129 pgs: 129 active+clean; 316 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T00:55:08.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 107024 0.72 MB/sec execute 317 sec latency 32.779 ms 2024-04-23T00:55:09.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:08.319784+0000 mgr.smithi153.olhmgd (mgr.24421) 246 : cluster [DBG] pgmap v173: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:09.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:08.319784+0000 mgr.smithi153.olhmgd (mgr.24421) 246 : cluster [DBG] pgmap v173: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:09.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 107504 0.74 MB/sec execute 318 sec latency 38.394 ms 2024-04-23T00:55:10.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:09.828016+0000 mon.smithi121 (mon.0) 811 : cluster [INF] MDS health message cleared (mds.?): Client smithi121:foo1 failing to respond to capability release 2024-04-23T00:55:10.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:09.828040+0000 mon.smithi121 (mon.0) 812 : cluster [INF] MDS health message cleared (mds.?): 1 slow requests are blocked > 30 secs 2024-04-23T00:55:10.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:09.828016+0000 mon.smithi121 (mon.0) 811 : cluster [INF] MDS health message cleared (mds.?): Client smithi121:foo1 failing to respond to capability release 2024-04-23T00:55:10.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:09.828040+0000 mon.smithi121 (mon.0) 812 : cluster [INF] MDS health message cleared (mds.?): 1 slow requests are blocked > 30 secs 2024-04-23T00:55:10.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 107986 0.77 MB/sec execute 319 sec latency 36.315 ms 2024-04-23T00:55:11.532 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.219509+0000 mon.smithi121 (mon.0) 813 : cluster [INF] Health check cleared: MDS_CLIENT_LATE_RELEASE (was: 1 clients failing to respond to capability release) 2024-04-23T00:55:11.532 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.219541+0000 mon.smithi121 (mon.0) 814 : cluster [INF] Health check cleared: MDS_SLOW_REQUEST (was: 1 MDSs report slow requests) 2024-04-23T00:55:11.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.219559+0000 mon.smithi121 (mon.0) 815 : cluster [INF] Cluster is now healthy 2024-04-23T00:55:11.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.230847+0000 mon.smithi121 (mon.0) 816 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:55:11.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.230911+0000 mon.smithi121 (mon.0) 817 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:55:11.533 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:10.320129+0000 mgr.smithi153.olhmgd (mgr.24421) 247 : cluster [DBG] pgmap v174: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:11.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.219509+0000 mon.smithi121 (mon.0) 813 : cluster [INF] Health check cleared: MDS_CLIENT_LATE_RELEASE (was: 1 clients failing to respond to capability release) 2024-04-23T00:55:11.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.219541+0000 mon.smithi121 (mon.0) 814 : cluster [INF] Health check cleared: MDS_SLOW_REQUEST (was: 1 MDSs report slow requests) 2024-04-23T00:55:11.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.219559+0000 mon.smithi121 (mon.0) 815 : cluster [INF] Cluster is now healthy 2024-04-23T00:55:11.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.230847+0000 mon.smithi121 (mon.0) 816 : cluster [DBG] mds.? [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] up:active 2024-04-23T00:55:11.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.230911+0000 mon.smithi121 (mon.0) 817 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T00:55:11.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:10.320129+0000 mgr.smithi153.olhmgd (mgr.24421) 247 : cluster [DBG] pgmap v174: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:11.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 108622 0.81 MB/sec execute 320 sec latency 36.087 ms 2024-04-23T00:55:12.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 109377 0.89 MB/sec execute 321 sec latency 26.483 ms 2024-04-23T00:55:13.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 110003 0.97 MB/sec execute 322 sec latency 36.338 ms 2024-04-23T00:55:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:12.320491+0000 mgr.smithi153.olhmgd (mgr.24421) 248 : cluster [DBG] pgmap v175: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:12.403699+0000 mon.smithi121 (mon.0) 818 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:55:13.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:12.404988+0000 mon.smithi153 (mon.1) 145 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:13.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:12.320491+0000 mgr.smithi153.olhmgd (mgr.24421) 248 : cluster [DBG] pgmap v175: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 288 KiB/s rd, 3.2 MiB/s wr, 275 op/s 2024-04-23T00:55:13.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:12.403699+0000 mon.smithi121 (mon.0) 818 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:55:13.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:12.404988+0000 mon.smithi153 (mon.1) 145 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:14.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 110454 1.00 MB/sec execute 323 sec latency 40.499 ms 2024-04-23T00:55:15.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 110808 1.01 MB/sec execute 324 sec latency 42.826 ms 2024-04-23T00:55:15.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:14.321055+0000 mgr.smithi153.olhmgd (mgr.24421) 249 : cluster [DBG] pgmap v176: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 594 KiB/s rd, 7.4 MiB/s wr, 721 op/s 2024-04-23T00:55:15.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:14.321055+0000 mgr.smithi153.olhmgd (mgr.24421) 249 : cluster [DBG] pgmap v176: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 594 KiB/s rd, 7.4 MiB/s wr, 721 op/s 2024-04-23T00:55:16.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 111231 1.04 MB/sec execute 325 sec latency 49.953 ms 2024-04-23T00:55:17.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 111759 1.06 MB/sec execute 326 sec latency 47.434 ms 2024-04-23T00:55:17.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:16.321410+0000 mgr.smithi153.olhmgd (mgr.24421) 250 : cluster [DBG] pgmap v177: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 498 KiB/s rd, 6.2 MiB/s wr, 604 op/s 2024-04-23T00:55:17.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:16.321410+0000 mgr.smithi153.olhmgd (mgr.24421) 250 : cluster [DBG] pgmap v177: 129 pgs: 129 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 498 KiB/s rd, 6.2 MiB/s wr, 604 op/s 2024-04-23T00:55:18.692 INFO:teuthology.orchestra.run.smithi121.stdout: 5 112808 1.15 MB/sec execute 327 sec latency 37.326 ms 2024-04-23T00:55:19.017 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:55:19.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:18.322118+0000 mgr.smithi153.olhmgd (mgr.24421) 251 : cluster [DBG] pgmap v178: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 500 KiB/s rd, 9.2 MiB/s wr, 932 op/s 2024-04-23T00:55:19.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:18.322118+0000 mgr.smithi153.olhmgd (mgr.24421) 251 : cluster [DBG] pgmap v178: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 500 KiB/s rd, 9.2 MiB/s wr, 932 op/s 2024-04-23T00:55:19.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 113728 1.25 MB/sec execute 328 sec latency 37.185 ms 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (13m) 4m ago 13m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (12m) 5m ago 12m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (7m) 4m ago 7m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:55:19.982 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (7m) 5m ago 7m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (14m) 4m ago 14m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (5m) 5m ago 12m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (14m) 4m ago 14m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (12m) 5m ago 12m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (5m) 4m ago 5m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (5m) 5m ago 5m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (12m) 4m ago 12m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (12m) 5m ago 12m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (11m) 4m ago 11m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (11m) 4m ago 11m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (10m) 4m ago 10m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (10m) 4m ago 10m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:55:19.983 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (10m) 5m ago 10m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:55:19.984 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (10m) 5m ago 10m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:55:19.984 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (9m) 5m ago 9m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:55:19.984 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (9m) 5m ago 9m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:55:19.984 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (11m) 4m ago 12m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:55:20.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:18.994993+0000 mgr.smithi153.olhmgd (mgr.24421) 252 : audit [DBG] from='client.14990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:20.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:19.498834+0000 mgr.smithi153.olhmgd (mgr.24421) 253 : audit [DBG] from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:20.587 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:55:20.587 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:55:20.588 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:55:20.589 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:55:20.589 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:55:20.589 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:55:20.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:18.994993+0000 mgr.smithi153.olhmgd (mgr.24421) 252 : audit [DBG] from='client.14990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:20.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:19.498834+0000 mgr.smithi153.olhmgd (mgr.24421) 253 : audit [DBG] from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:20.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 114407 1.29 MB/sec execute 329 sec latency 34.063 ms 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:55:21.095 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:55:21.531 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:19.965265+0000 mgr.smithi153.olhmgd (mgr.24421) 254 : audit [DBG] from='client.24681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:21.531 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:20.322493+0000 mgr.smithi153.olhmgd (mgr.24421) 255 : cluster [DBG] pgmap v179: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 261 KiB/s rd, 6.5 MiB/s wr, 702 op/s 2024-04-23T00:55:21.531 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:20.584527+0000 mon.smithi121 (mon.0) 819 : audit [DBG] from='client.? 172.21.15.121:0/243280710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:55:21.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:19.965265+0000 mgr.smithi153.olhmgd (mgr.24421) 254 : audit [DBG] from='client.24681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:21.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:20.322493+0000 mgr.smithi153.olhmgd (mgr.24421) 255 : cluster [DBG] pgmap v179: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 261 KiB/s rd, 6.5 MiB/s wr, 702 op/s 2024-04-23T00:55:21.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:20.584527+0000 mon.smithi121 (mon.0) 819 : audit [DBG] from='client.? 172.21.15.121:0/243280710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:55:21.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 115137 1.33 MB/sec execute 330 sec latency 28.726 ms 2024-04-23T00:55:21.706 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:55:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:21.091389+0000 mgr.smithi153.olhmgd (mgr.24421) 256 : audit [DBG] from='client.15002 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:22.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:21.700732+0000 mon.smithi153 (mon.1) 146 : audit [DBG] from='client.? 172.21.15.121:0/2199079343' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:55:22.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:21.091389+0000 mgr.smithi153.olhmgd (mgr.24421) 256 : audit [DBG] from='client.15002 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:22.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:21.700732+0000 mon.smithi153 (mon.1) 146 : audit [DBG] from='client.? 172.21.15.121:0/2199079343' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:55:22.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 115766 1.36 MB/sec execute 331 sec latency 19.937 ms 2024-04-23T00:55:23.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:22.322828+0000 mgr.smithi153.olhmgd (mgr.24421) 257 : cluster [DBG] pgmap v180: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 261 KiB/s rd, 6.5 MiB/s wr, 702 op/s 2024-04-23T00:55:23.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:22.322828+0000 mgr.smithi153.olhmgd (mgr.24421) 257 : cluster [DBG] pgmap v180: 129 pgs: 129 active+clean; 323 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 261 KiB/s rd, 6.5 MiB/s wr, 702 op/s 2024-04-23T00:55:23.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 116216 1.40 MB/sec execute 332 sec latency 43.001 ms 2024-04-23T00:55:24.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 116659 1.46 MB/sec execute 333 sec latency 58.475 ms 2024-04-23T00:55:25.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 117105 1.51 MB/sec execute 334 sec latency 34.378 ms 2024-04-23T00:55:25.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:24.323569+0000 mgr.smithi153.olhmgd (mgr.24421) 258 : cluster [DBG] pgmap v181: 129 pgs: 129 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 266 KiB/s rd, 11 MiB/s wr, 1.14k op/s 2024-04-23T00:55:25.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:24.323569+0000 mgr.smithi153.olhmgd (mgr.24421) 258 : cluster [DBG] pgmap v181: 129 pgs: 129 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 266 KiB/s rd, 11 MiB/s wr, 1.14k op/s 2024-04-23T00:55:26.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 117531 1.53 MB/sec execute 335 sec latency 34.218 ms 2024-04-23T00:55:27.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 117888 1.54 MB/sec execute 336 sec latency 68.106 ms 2024-04-23T00:55:27.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:26.323897+0000 mgr.smithi153.olhmgd (mgr.24421) 259 : cluster [DBG] pgmap v182: 129 pgs: 129 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 766 op/s 2024-04-23T00:55:27.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:26.323897+0000 mgr.smithi153.olhmgd (mgr.24421) 259 : cluster [DBG] pgmap v182: 129 pgs: 129 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 766 op/s 2024-04-23T00:55:28.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 118324 1.57 MB/sec execute 337 sec latency 43.384 ms 2024-04-23T00:55:28.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:27.400826+0000 mon.smithi153 (mon.1) 147 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:28.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:27.400826+0000 mon.smithi153 (mon.1) 147 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:29.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 118879 1.60 MB/sec execute 338 sec latency 33.854 ms 2024-04-23T00:55:29.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:28.324560+0000 mgr.smithi153.olhmgd (mgr.24421) 260 : cluster [DBG] pgmap v183: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-04-23T00:55:29.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:28.324560+0000 mgr.smithi153.olhmgd (mgr.24421) 260 : cluster [DBG] pgmap v183: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-04-23T00:55:30.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 119428 1.63 MB/sec execute 339 sec latency 34.004 ms 2024-04-23T00:55:31.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 120022 1.68 MB/sec execute 340 sec latency 25.892 ms 2024-04-23T00:55:31.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:30.324878+0000 mgr.smithi153.olhmgd (mgr.24421) 261 : cluster [DBG] pgmap v184: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 706 op/s 2024-04-23T00:55:31.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:30.324878+0000 mgr.smithi153.olhmgd (mgr.24421) 261 : cluster [DBG] pgmap v184: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 706 op/s 2024-04-23T00:55:32.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 120464 1.73 MB/sec execute 341 sec latency 47.263 ms 2024-04-23T00:55:33.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 120947 1.77 MB/sec execute 342 sec latency 30.213 ms 2024-04-23T00:55:33.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:32.325203+0000 mgr.smithi153.olhmgd (mgr.24421) 262 : cluster [DBG] pgmap v185: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 706 op/s 2024-04-23T00:55:33.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:32.325203+0000 mgr.smithi153.olhmgd (mgr.24421) 262 : cluster [DBG] pgmap v185: 129 pgs: 129 active+clean; 324 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 706 op/s 2024-04-23T00:55:34.693 INFO:teuthology.orchestra.run.smithi121.stdout: 5 121501 1.79 MB/sec execute 343 sec latency 56.271 ms 2024-04-23T00:55:35.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 122498 1.85 MB/sec execute 344 sec latency 26.505 ms 2024-04-23T00:55:35.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:34.325838+0000 mgr.smithi153.olhmgd (mgr.24421) 263 : cluster [DBG] pgmap v186: 129 pgs: 129 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-23T00:55:35.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:34.325838+0000 mgr.smithi153.olhmgd (mgr.24421) 263 : cluster [DBG] pgmap v186: 129 pgs: 129 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-23T00:55:36.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 123382 1.91 MB/sec execute 345 sec latency 17.311 ms 2024-04-23T00:55:37.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 124068 1.99 MB/sec execute 346 sec latency 31.522 ms 2024-04-23T00:55:37.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:36.326182+0000 mgr.smithi153.olhmgd (mgr.24421) 264 : cluster [DBG] pgmap v187: 129 pgs: 129 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-04-23T00:55:37.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:36.326182+0000 mgr.smithi153.olhmgd (mgr.24421) 264 : cluster [DBG] pgmap v187: 129 pgs: 129 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-04-23T00:55:38.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 124477 2.02 MB/sec execute 347 sec latency 32.227 ms 2024-04-23T00:55:39.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:38.326838+0000 mgr.smithi153.olhmgd (mgr.24421) 265 : cluster [DBG] pgmap v188: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 998 op/s 2024-04-23T00:55:39.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:38.326838+0000 mgr.smithi153.olhmgd (mgr.24421) 265 : cluster [DBG] pgmap v188: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 998 op/s 2024-04-23T00:55:39.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 124892 2.03 MB/sec execute 348 sec latency 43.578 ms 2024-04-23T00:55:40.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 125800 2.08 MB/sec execute 349 sec latency 79.037 ms 2024-04-23T00:55:41.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 126938 2.16 MB/sec execute 350 sec latency 14.262 ms 2024-04-23T00:55:41.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:40.327456+0000 mgr.smithi153.olhmgd (mgr.24421) 266 : cluster [DBG] pgmap v189: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-04-23T00:55:41.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:40.327456+0000 mgr.smithi153.olhmgd (mgr.24421) 266 : cluster [DBG] pgmap v189: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-04-23T00:55:42.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 127767 2.25 MB/sec execute 351 sec latency 34.844 ms 2024-04-23T00:55:43.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 128108 2.27 MB/sec execute 352 sec latency 36.756 ms 2024-04-23T00:55:43.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:42.327842+0000 mgr.smithi153.olhmgd (mgr.24421) 267 : cluster [DBG] pgmap v190: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-04-23T00:55:43.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:42.401186+0000 mon.smithi153 (mon.1) 148 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:43.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:42.327842+0000 mgr.smithi153.olhmgd (mgr.24421) 267 : cluster [DBG] pgmap v190: 129 pgs: 129 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 727 op/s 2024-04-23T00:55:43.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:42.401186+0000 mon.smithi153 (mon.1) 148 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:44.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 128438 2.27 MB/sec execute 353 sec latency 66.485 ms 2024-04-23T00:55:45.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 129328 2.33 MB/sec execute 354 sec latency 27.942 ms 2024-04-23T00:55:45.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:44.328584+0000 mgr.smithi153.olhmgd (mgr.24421) 268 : cluster [DBG] pgmap v191: 129 pgs: 129 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-23T00:55:45.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:44.328584+0000 mgr.smithi153.olhmgd (mgr.24421) 268 : cluster [DBG] pgmap v191: 129 pgs: 129 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-23T00:55:46.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 130132 2.37 MB/sec execute 355 sec latency 30.793 ms 2024-04-23T00:55:47.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 130639 2.41 MB/sec execute 356 sec latency 30.567 ms 2024-04-23T00:55:47.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:46.328930+0000 mgr.smithi153.olhmgd (mgr.24421) 269 : cluster [DBG] pgmap v192: 129 pgs: 129 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 843 op/s 2024-04-23T00:55:47.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:46.328930+0000 mgr.smithi153.olhmgd (mgr.24421) 269 : cluster [DBG] pgmap v192: 129 pgs: 129 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 843 op/s 2024-04-23T00:55:48.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 131089 2.46 MB/sec execute 357 sec latency 48.378 ms 2024-04-23T00:55:49.575 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:48.329514+0000 mgr.smithi153.olhmgd (mgr.24421) 270 : cluster [DBG] pgmap v193: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-04-23T00:55:49.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:48.329514+0000 mgr.smithi153.olhmgd (mgr.24421) 270 : cluster [DBG] pgmap v193: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-04-23T00:55:49.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 131703 2.50 MB/sec execute 358 sec latency 34.581 ms 2024-04-23T00:55:50.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 132186 2.51 MB/sec execute 359 sec latency 53.736 ms 2024-04-23T00:55:51.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 132544 2.53 MB/sec execute 360 sec latency 47.809 ms 2024-04-23T00:55:51.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:50.331604+0000 mgr.smithi153.olhmgd (mgr.24421) 271 : cluster [DBG] pgmap v194: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-04-23T00:55:51.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:50.331604+0000 mgr.smithi153.olhmgd (mgr.24421) 271 : cluster [DBG] pgmap v194: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-04-23T00:55:52.082 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:55:52.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 132937 2.55 MB/sec execute 361 sec latency 56.699 ms 2024-04-23T00:55:52.784 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (13m) 5m ago 13m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (13m) 5m ago 13m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (8m) 5m ago 8m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (8m) 5m ago 8m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (15m) 5m ago 15m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (6m) 5m ago 13m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (15m) 5m ago 15m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (12m) 5m ago 12m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (5m) 5m ago 5m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:55:52.785 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (5m) 5m ago 5m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (13m) 5m ago 13m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (12m) 5m ago 12m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (12m) 5m ago 12m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (11m) 5m ago 11m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (11m) 5m ago 11m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (11m) 5m ago 11m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (10m) 5m ago 10m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (10m) 5m ago 10m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (10m) 5m ago 10m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (9m) 5m ago 9m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:55:52.786 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (12m) 5m ago 13m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:55:53.219 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:55:53.219 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:55:53.220 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:55:53.221 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:55:53.221 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:55:53.221 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:55:53.221 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:55:53.570 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:55:53.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 133317 2.55 MB/sec execute 362 sec latency 56.526 ms 2024-04-23T00:55:53.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:52.064437+0000 mgr.smithi153.olhmgd (mgr.24421) 272 : audit [DBG] from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:52.331934+0000 mgr.smithi153.olhmgd (mgr.24421) 273 : cluster [DBG] pgmap v195: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-04-23T00:55:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:52.419242+0000 mgr.smithi153.olhmgd (mgr.24421) 274 : audit [DBG] from='client.15010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:52.767532+0000 mgr.smithi153.olhmgd (mgr.24421) 275 : audit [DBG] from='client.15014 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:53.215630+0000 mon.smithi121 (mon.0) 820 : audit [DBG] from='client.? 172.21.15.121:0/1269079758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:55:53.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:52.064437+0000 mgr.smithi153.olhmgd (mgr.24421) 272 : audit [DBG] from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:52.331934+0000 mgr.smithi153.olhmgd (mgr.24421) 273 : cluster [DBG] pgmap v195: 129 pgs: 129 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-04-23T00:55:53.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:52.419242+0000 mgr.smithi153.olhmgd (mgr.24421) 274 : audit [DBG] from='client.15010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:52.767532+0000 mgr.smithi153.olhmgd (mgr.24421) 275 : audit [DBG] from='client.15014 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:53.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:53.215630+0000 mon.smithi121 (mon.0) 820 : audit [DBG] from='client.? 172.21.15.121:0/1269079758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:55:54.083 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:55:54.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 133722 2.58 MB/sec execute 363 sec latency 49.669 ms 2024-04-23T00:55:54.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:53.566245+0000 mgr.smithi153.olhmgd (mgr.24421) 276 : audit [DBG] from='client.15022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:54.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:54.079572+0000 mon.smithi121 (mon.0) 821 : audit [DBG] from='client.? 172.21.15.121:0/2653132683' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:55:54.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:53.566245+0000 mgr.smithi153.olhmgd (mgr.24421) 276 : audit [DBG] from='client.15022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:55:54.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:54.079572+0000 mon.smithi121 (mon.0) 821 : audit [DBG] from='client.? 172.21.15.121:0/2653132683' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:55:55.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 134225 2.62 MB/sec execute 364 sec latency 34.461 ms 2024-04-23T00:55:55.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:54.332588+0000 mgr.smithi153.olhmgd (mgr.24421) 277 : cluster [DBG] pgmap v196: 129 pgs: 129 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.05k op/s 2024-04-23T00:55:55.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:54.332588+0000 mgr.smithi153.olhmgd (mgr.24421) 277 : cluster [DBG] pgmap v196: 129 pgs: 129 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.05k op/s 2024-04-23T00:55:56.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 134639 2.66 MB/sec execute 365 sec latency 75.292 ms 2024-04-23T00:55:57.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 135005 2.69 MB/sec execute 366 sec latency 29.116 ms 2024-04-23T00:55:57.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:56.332901+0000 mgr.smithi153.olhmgd (mgr.24421) 278 : cluster [DBG] pgmap v197: 129 pgs: 129 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 628 op/s 2024-04-23T00:55:57.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:56.332901+0000 mgr.smithi153.olhmgd (mgr.24421) 278 : cluster [DBG] pgmap v197: 129 pgs: 129 active+clean; 324 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 628 op/s 2024-04-23T00:55:58.694 INFO:teuthology.orchestra.run.smithi121.stdout: 5 135321 2.70 MB/sec execute 367 sec latency 58.644 ms 2024-04-23T00:55:58.825 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:55:57.403174+0000 mon.smithi153 (mon.1) 149 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:58.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:55:57.403174+0000 mon.smithi153 (mon.1) 149 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:55:59.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 135760 2.71 MB/sec execute 368 sec latency 35.973 ms 2024-04-23T00:55:59.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:55:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:55:58.333581+0000 mgr.smithi153.olhmgd (mgr.24421) 279 : cluster [DBG] pgmap v198: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-04-23T00:55:59.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:55:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:55:58.333581+0000 mgr.smithi153.olhmgd (mgr.24421) 279 : cluster [DBG] pgmap v198: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-04-23T00:56:00.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 136447 2.75 MB/sec execute 369 sec latency 32.351 ms 2024-04-23T00:56:01.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 136921 2.76 MB/sec execute 370 sec latency 33.857 ms 2024-04-23T00:56:01.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:00.333933+0000 mgr.smithi153.olhmgd (mgr.24421) 280 : cluster [DBG] pgmap v199: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-04-23T00:56:01.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:00.333933+0000 mgr.smithi153.olhmgd (mgr.24421) 280 : cluster [DBG] pgmap v199: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-04-23T00:56:02.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 137265 2.78 MB/sec execute 371 sec latency 60.390 ms 2024-04-23T00:56:03.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 137681 2.82 MB/sec execute 372 sec latency 35.907 ms 2024-04-23T00:56:03.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:02.334321+0000 mgr.smithi153.olhmgd (mgr.24421) 281 : cluster [DBG] pgmap v200: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-04-23T00:56:03.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:02.334321+0000 mgr.smithi153.olhmgd (mgr.24421) 281 : cluster [DBG] pgmap v200: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-04-23T00:56:04.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 138037 2.84 MB/sec execute 373 sec latency 90.011 ms 2024-04-23T00:56:05.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 138495 2.89 MB/sec execute 374 sec latency 30.993 ms 2024-04-23T00:56:05.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:04.335068+0000 mgr.smithi153.olhmgd (mgr.24421) 282 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 823 op/s 2024-04-23T00:56:05.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:04.335068+0000 mgr.smithi153.olhmgd (mgr.24421) 282 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 823 op/s 2024-04-23T00:56:06.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 138847 2.90 MB/sec execute 375 sec latency 39.611 ms 2024-04-23T00:56:07.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 139230 2.90 MB/sec execute 376 sec latency 43.558 ms 2024-04-23T00:56:07.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:06.335461+0000 mgr.smithi153.olhmgd (mgr.24421) 283 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.4 MiB/s wr, 535 op/s 2024-04-23T00:56:07.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:06.335461+0000 mgr.smithi153.olhmgd (mgr.24421) 283 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.4 MiB/s wr, 535 op/s 2024-04-23T00:56:08.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 139742 2.93 MB/sec execute 377 sec latency 33.445 ms 2024-04-23T00:56:09.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:08.336102+0000 mgr.smithi153.olhmgd (mgr.24421) 284 : cluster [DBG] pgmap v203: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 799 op/s 2024-04-23T00:56:09.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:08.336102+0000 mgr.smithi153.olhmgd (mgr.24421) 284 : cluster [DBG] pgmap v203: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 799 op/s 2024-04-23T00:56:09.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 140181 2.94 MB/sec execute 378 sec latency 59.128 ms 2024-04-23T00:56:10.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 140581 2.96 MB/sec execute 379 sec latency 35.483 ms 2024-04-23T00:56:11.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 140930 2.98 MB/sec execute 380 sec latency 58.895 ms 2024-04-23T00:56:11.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:10.336428+0000 mgr.smithi153.olhmgd (mgr.24421) 285 : cluster [DBG] pgmap v204: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-23T00:56:11.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:10.336428+0000 mgr.smithi153.olhmgd (mgr.24421) 285 : cluster [DBG] pgmap v204: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-23T00:56:12.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 141340 3.01 MB/sec execute 381 sec latency 35.641 ms 2024-04-23T00:56:13.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 141862 3.06 MB/sec execute 382 sec latency 33.657 ms 2024-04-23T00:56:13.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:12.336812+0000 mgr.smithi153.olhmgd (mgr.24421) 286 : cluster [DBG] pgmap v205: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-23T00:56:13.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:12.401839+0000 mon.smithi153 (mon.1) 150 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:13.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:12.336812+0000 mgr.smithi153.olhmgd (mgr.24421) 286 : cluster [DBG] pgmap v205: 129 pgs: 129 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-04-23T00:56:13.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:12.401839+0000 mon.smithi153 (mon.1) 150 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:14.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 142252 3.08 MB/sec execute 383 sec latency 58.685 ms 2024-04-23T00:56:15.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 142571 3.08 MB/sec execute 384 sec latency 56.804 ms 2024-04-23T00:56:15.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:14.337510+0000 mgr.smithi153.olhmgd (mgr.24421) 287 : cluster [DBG] pgmap v206: 129 pgs: 129 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 798 op/s 2024-04-23T00:56:15.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:14.337510+0000 mgr.smithi153.olhmgd (mgr.24421) 287 : cluster [DBG] pgmap v206: 129 pgs: 129 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 798 op/s 2024-04-23T00:56:16.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 142852 3.09 MB/sec execute 385 sec latency 52.023 ms 2024-04-23T00:56:17.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 143279 3.11 MB/sec execute 386 sec latency 45.516 ms 2024-04-23T00:56:17.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:16.337817+0000 mgr.smithi153.olhmgd (mgr.24421) 288 : cluster [DBG] pgmap v207: 129 pgs: 129 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 505 op/s 2024-04-23T00:56:17.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:16.337817+0000 mgr.smithi153.olhmgd (mgr.24421) 288 : cluster [DBG] pgmap v207: 129 pgs: 129 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 505 op/s 2024-04-23T00:56:18.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 143793 3.12 MB/sec execute 387 sec latency 38.014 ms 2024-04-23T00:56:19.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:18.338397+0000 mgr.smithi153.olhmgd (mgr.24421) 289 : cluster [DBG] pgmap v208: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 760 op/s 2024-04-23T00:56:19.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:18.338397+0000 mgr.smithi153.olhmgd (mgr.24421) 289 : cluster [DBG] pgmap v208: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 760 op/s 2024-04-23T00:56:19.695 INFO:teuthology.orchestra.run.smithi121.stdout: 5 144300 3.15 MB/sec execute 388 sec latency 39.020 ms 2024-04-23T00:56:20.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 144965 3.20 MB/sec execute 389 sec latency 30.294 ms 2024-04-23T00:56:21.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 145795 3.27 MB/sec execute 390 sec latency 18.249 ms 2024-04-23T00:56:21.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:20.338686+0000 mgr.smithi153.olhmgd (mgr.24421) 290 : cluster [DBG] pgmap v209: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 497 op/s 2024-04-23T00:56:21.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:20.338686+0000 mgr.smithi153.olhmgd (mgr.24421) 290 : cluster [DBG] pgmap v209: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 497 op/s 2024-04-23T00:56:22.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 146477 3.30 MB/sec execute 391 sec latency 35.282 ms 2024-04-23T00:56:23.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 147542 3.34 MB/sec execute 392 sec latency 23.733 ms 2024-04-23T00:56:23.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:22.339723+0000 mgr.smithi153.olhmgd (mgr.24421) 291 : cluster [DBG] pgmap v210: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 497 op/s 2024-04-23T00:56:23.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:22.339723+0000 mgr.smithi153.olhmgd (mgr.24421) 291 : cluster [DBG] pgmap v210: 129 pgs: 129 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 497 op/s 2024-04-23T00:56:24.593 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:56:24.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 148402 3.40 MB/sec execute 393 sec latency 19.287 ms 2024-04-23T00:56:25.509 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (14m) 6m ago 14m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (13m) 6m ago 13m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (8m) 6m ago 8m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (8m) 6m ago 8m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (15m) 6m ago 15m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (6m) 6m ago 13m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (15m) 6m ago 16m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (13m) 6m ago 13m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (6m) 6m ago 6m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:56:25.510 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (6m) 6m ago 6m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (14m) 6m ago 14m 17.3M - 0.18.1 e5a616e4b9cf c8cfcb06b2ed 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (13m) 6m ago 13m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (12m) 6m ago 12m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (12m) 6m ago 12m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (12m) 6m ago 12m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (11m) 6m ago 11m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (11m) 6m ago 11m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (11m) 6m ago 11m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (10m) 6m ago 10m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (10m) 6m ago 10m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:56:25.511 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (12m) 6m ago 13m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:56:25.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 149202 3.47 MB/sec execute 394 sec latency 32.876 ms 2024-04-23T00:56:25.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:24.340354+0000 mgr.smithi153.olhmgd (mgr.24421) 292 : cluster [DBG] pgmap v211: 129 pgs: 129 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 919 op/s 2024-04-23T00:56:25.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:24.571132+0000 mgr.smithi153.olhmgd (mgr.24421) 293 : audit [DBG] from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:25.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:24.340354+0000 mgr.smithi153.olhmgd (mgr.24421) 292 : cluster [DBG] pgmap v211: 129 pgs: 129 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 919 op/s 2024-04-23T00:56:25.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:24.571132+0000 mgr.smithi153.olhmgd (mgr.24421) 293 : audit [DBG] from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:26.031 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:56:26.032 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:56:26.476 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:56:26.477 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:56:26.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 149875 3.49 MB/sec execute 395 sec latency 26.439 ms 2024-04-23T00:56:26.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:25.039851+0000 mgr.smithi153.olhmgd (mgr.24421) 294 : audit [DBG] from='client.24723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:26.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:25.490169+0000 mgr.smithi153.olhmgd (mgr.24421) 295 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:26.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:26 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:26.027053+0000 mon.smithi153 (mon.1) 151 : audit [DBG] from='client.? 172.21.15.121:0/3263945194' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:56:26.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:25.039851+0000 mgr.smithi153.olhmgd (mgr.24421) 294 : audit [DBG] from='client.24723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:26.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:25.490169+0000 mgr.smithi153.olhmgd (mgr.24421) 295 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:26.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:26 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:26.027053+0000 mon.smithi153 (mon.1) 151 : audit [DBG] from='client.? 172.21.15.121:0/3263945194' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:56:27.013 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:56:27.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 150686 3.53 MB/sec execute 396 sec latency 26.863 ms 2024-04-23T00:56:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:26.340700+0000 mgr.smithi153.olhmgd (mgr.24421) 296 : cluster [DBG] pgmap v212: 129 pgs: 129 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-04-23T00:56:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:26.470413+0000 mgr.smithi153.olhmgd (mgr.24421) 297 : audit [DBG] from='client.15044 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:27.009770+0000 mon.smithi121 (mon.0) 822 : audit [DBG] from='client.? 172.21.15.121:0/1140642400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:56:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:27.401999+0000 mon.smithi153 (mon.1) 152 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:26.340700+0000 mgr.smithi153.olhmgd (mgr.24421) 296 : cluster [DBG] pgmap v212: 129 pgs: 129 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-04-23T00:56:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:26.470413+0000 mgr.smithi153.olhmgd (mgr.24421) 297 : audit [DBG] from='client.15044 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:27.009770+0000 mon.smithi121 (mon.0) 822 : audit [DBG] from='client.? 172.21.15.121:0/1140642400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:56:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:27.401999+0000 mon.smithi153 (mon.1) 152 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:28.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 151394 3.56 MB/sec execute 397 sec latency 30.724 ms 2024-04-23T00:56:29.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:28.341362+0000 mgr.smithi153.olhmgd (mgr.24421) 298 : cluster [DBG] pgmap v213: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.13k op/s 2024-04-23T00:56:29.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:28.341362+0000 mgr.smithi153.olhmgd (mgr.24421) 298 : cluster [DBG] pgmap v213: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.13k op/s 2024-04-23T00:56:29.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 152050 3.61 MB/sec execute 398 sec latency 32.037 ms 2024-04-23T00:56:30.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 152784 3.67 MB/sec execute 399 sec latency 37.246 ms 2024-04-23T00:56:31.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 153313 3.68 MB/sec execute 400 sec latency 46.754 ms 2024-04-23T00:56:31.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:30.341754+0000 mgr.smithi153.olhmgd (mgr.24421) 299 : cluster [DBG] pgmap v214: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 873 op/s 2024-04-23T00:56:31.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:30.341754+0000 mgr.smithi153.olhmgd (mgr.24421) 299 : cluster [DBG] pgmap v214: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 873 op/s 2024-04-23T00:56:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 153609 3.69 MB/sec execute 401 sec latency 59.784 ms 2024-04-23T00:56:33.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 154007 3.70 MB/sec execute 402 sec latency 39.293 ms 2024-04-23T00:56:33.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:32.343646+0000 mgr.smithi153.olhmgd (mgr.24421) 300 : cluster [DBG] pgmap v215: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 873 op/s 2024-04-23T00:56:33.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:32.343646+0000 mgr.smithi153.olhmgd (mgr.24421) 300 : cluster [DBG] pgmap v215: 129 pgs: 129 active+clean; 329 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 873 op/s 2024-04-23T00:56:34.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 154662 3.73 MB/sec execute 403 sec latency 49.245 ms 2024-04-23T00:56:35.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 155438 3.78 MB/sec execute 404 sec latency 21.391 ms 2024-04-23T00:56:35.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:34.344266+0000 mgr.smithi153.olhmgd (mgr.24421) 301 : cluster [DBG] pgmap v216: 129 pgs: 129 active+clean; 323 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-04-23T00:56:35.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:34.344266+0000 mgr.smithi153.olhmgd (mgr.24421) 301 : cluster [DBG] pgmap v216: 129 pgs: 129 active+clean; 323 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-04-23T00:56:36.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 156121 3.84 MB/sec execute 405 sec latency 23.783 ms 2024-04-23T00:56:37.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 156653 3.86 MB/sec execute 406 sec latency 39.969 ms 2024-04-23T00:56:37.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:36.344580+0000 mgr.smithi153.olhmgd (mgr.24421) 302 : cluster [DBG] pgmap v217: 129 pgs: 129 active+clean; 323 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 785 op/s 2024-04-23T00:56:37.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:36.344580+0000 mgr.smithi153.olhmgd (mgr.24421) 302 : cluster [DBG] pgmap v217: 129 pgs: 129 active+clean; 323 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 785 op/s 2024-04-23T00:56:38.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 157087 3.87 MB/sec execute 407 sec latency 51.125 ms 2024-04-23T00:56:39.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:38.345777+0000 mgr.smithi153.olhmgd (mgr.24421) 303 : cluster [DBG] pgmap v218: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-23T00:56:39.630 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:38.345777+0000 mgr.smithi153.olhmgd (mgr.24421) 303 : cluster [DBG] pgmap v218: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-04-23T00:56:39.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 157763 3.89 MB/sec execute 408 sec latency 36.344 ms 2024-04-23T00:56:40.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 158322 3.91 MB/sec execute 409 sec latency 27.124 ms 2024-04-23T00:56:41.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 158886 3.95 MB/sec execute 410 sec latency 41.954 ms 2024-04-23T00:56:41.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:40.346098+0000 mgr.smithi153.olhmgd (mgr.24421) 304 : cluster [DBG] pgmap v219: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-04-23T00:56:41.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:40.346098+0000 mgr.smithi153.olhmgd (mgr.24421) 304 : cluster [DBG] pgmap v219: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-04-23T00:56:42.696 INFO:teuthology.orchestra.run.smithi121.stdout: 5 159601 4.01 MB/sec execute 411 sec latency 32.683 ms 2024-04-23T00:56:43.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 160280 4.03 MB/sec execute 412 sec latency 29.346 ms 2024-04-23T00:56:43.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:42.346406+0000 mgr.smithi153.olhmgd (mgr.24421) 305 : cluster [DBG] pgmap v220: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-04-23T00:56:43.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:42.402985+0000 mon.smithi153 (mon.1) 153 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:43.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:42.346406+0000 mgr.smithi153.olhmgd (mgr.24421) 305 : cluster [DBG] pgmap v220: 129 pgs: 129 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-04-23T00:56:43.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:42.402985+0000 mon.smithi153 (mon.1) 153 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:44.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 160832 4.05 MB/sec execute 413 sec latency 33.290 ms 2024-04-23T00:56:45.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 161583 4.08 MB/sec execute 414 sec latency 31.491 ms 2024-04-23T00:56:45.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:44.347255+0000 mgr.smithi153.olhmgd (mgr.24421) 306 : cluster [DBG] pgmap v221: 129 pgs: 129 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-23T00:56:45.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:44.347255+0000 mgr.smithi153.olhmgd (mgr.24421) 306 : cluster [DBG] pgmap v221: 129 pgs: 129 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-23T00:56:46.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 162170 4.10 MB/sec execute 415 sec latency 23.472 ms 2024-04-23T00:56:47.412 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:46.347622+0000 mgr.smithi153.olhmgd (mgr.24421) 307 : cluster [DBG] pgmap v222: 129 pgs: 129 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 729 op/s 2024-04-23T00:56:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 162752 4.15 MB/sec execute 416 sec latency 51.072 ms 2024-04-23T00:56:47.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:46.347622+0000 mgr.smithi153.olhmgd (mgr.24421) 307 : cluster [DBG] pgmap v222: 129 pgs: 129 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 729 op/s 2024-04-23T00:56:48.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 163447 4.20 MB/sec execute 417 sec latency 30.102 ms 2024-04-23T00:56:49.243 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:48.348309+0000 mgr.smithi153.olhmgd (mgr.24421) 308 : cluster [DBG] pgmap v223: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-04-23T00:56:49.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:48.348309+0000 mgr.smithi153.olhmgd (mgr.24421) 308 : cluster [DBG] pgmap v223: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-04-23T00:56:49.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 163945 4.21 MB/sec execute 418 sec latency 36.458 ms 2024-04-23T00:56:50.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 164360 4.22 MB/sec execute 419 sec latency 42.672 ms 2024-04-23T00:56:51.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:50.063801+0000 mon.smithi121 (mon.0) 823 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:56:51.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:50.069918+0000 mon.smithi121 (mon.0) 824 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:56:51.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:56:50.088960+0000 mgr.smithi153.olhmgd (mgr.24421) 309 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T00:56:51.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:56:50.328165+0000 mgr.smithi153.olhmgd (mgr.24421) 310 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T00:56:51.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:50.348690+0000 mgr.smithi153.olhmgd (mgr.24421) 311 : cluster [DBG] pgmap v224: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 732 op/s 2024-04-23T00:56:51.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:50.063801+0000 mon.smithi121 (mon.0) 823 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:56:51.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:50.069918+0000 mon.smithi121 (mon.0) 824 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:56:51.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:56:50.088960+0000 mgr.smithi153.olhmgd (mgr.24421) 309 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T00:56:51.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:56:50.328165+0000 mgr.smithi153.olhmgd (mgr.24421) 310 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T00:56:51.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:50.348690+0000 mgr.smithi153.olhmgd (mgr.24421) 311 : cluster [DBG] pgmap v224: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 732 op/s 2024-04-23T00:56:51.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 165051 4.24 MB/sec execute 420 sec latency 59.230 ms 2024-04-23T00:56:52.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 165850 4.28 MB/sec execute 421 sec latency 17.291 ms 2024-04-23T00:56:53.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 166617 4.35 MB/sec execute 422 sec latency 28.823 ms 2024-04-23T00:56:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:52.349086+0000 mgr.smithi153.olhmgd (mgr.24421) 312 : cluster [DBG] pgmap v225: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 732 op/s 2024-04-23T00:56:53.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:52.349086+0000 mgr.smithi153.olhmgd (mgr.24421) 312 : cluster [DBG] pgmap v225: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.5 MiB/s wr, 732 op/s 2024-04-23T00:56:54.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 167211 4.38 MB/sec execute 423 sec latency 38.977 ms 2024-04-23T00:56:55.697 INFO:teuthology.orchestra.run.smithi121.stdout: 5 167621 4.38 MB/sec execute 424 sec latency 37.923 ms 2024-04-23T00:56:55.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:54.349720+0000 mgr.smithi153.olhmgd (mgr.24421) 313 : cluster [DBG] pgmap v226: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-04-23T00:56:55.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:54.349720+0000 mgr.smithi153.olhmgd (mgr.24421) 313 : cluster [DBG] pgmap v226: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.10k op/s 2024-04-23T00:56:56.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 168190 4.40 MB/sec execute 425 sec latency 44.110 ms 2024-04-23T00:56:57.498 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:56:57.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 168722 4.41 MB/sec execute 426 sec latency 32.508 ms 2024-04-23T00:56:57.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:56.350087+0000 mgr.smithi153.olhmgd (mgr.24421) 314 : cluster [DBG] pgmap v227: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-23T00:56:57.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:57.402741+0000 mon.smithi153 (mon.1) 154 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:57.880 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:56.350087+0000 mgr.smithi153.olhmgd (mgr.24421) 314 : cluster [DBG] pgmap v227: 129 pgs: 129 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-23T00:56:57.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:57.402741+0000 mon.smithi153 (mon.1) 154 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (15m) 6m ago 15m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (14m) 6m ago 14m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:56:58.425 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (9m) 6m ago 9m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (9m) 6m ago 9m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (16m) 6m ago 16m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (7m) 6m ago 14m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (16m) 6m ago 16m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (14m) 6m ago 14m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (6m) 6m ago 6m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (6m) 6m ago 6m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (13m) 6m ago 13m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (13m) 6m ago 13m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (13m) 6m ago 13m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (12m) 6m ago 12m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:56:58.426 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (12m) 6m ago 12m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:56:58.427 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (11m) 6m ago 11m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:56:58.427 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (11m) 6m ago 11m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:56:58.427 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (11m) 6m ago 11m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:56:58.427 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (10m) 6m ago 10m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:56:58.427 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (13m) 6m ago 14m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:56:58.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 169125 4.43 MB/sec execute 427 sec latency 38.838 ms 2024-04-23T00:56:58.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:57.473527+0000 mgr.smithi153.olhmgd (mgr.24421) 315 : audit [DBG] from='client.15052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:58.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:58 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:57.473527+0000 mgr.smithi153.olhmgd (mgr.24421) 315 : audit [DBG] from='client.15052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:56:58.970 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:56:58.971 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:56:59.439 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:56:59.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:57.954463+0000 mgr.smithi153.olhmgd (mgr.24421) 316 : audit [DBG] from='client.24741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:59.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:56:58.350662+0000 mgr.smithi153.olhmgd (mgr.24421) 317 : cluster [DBG] pgmap v228: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-04-23T00:56:59.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:58.408469+0000 mgr.smithi153.olhmgd (mgr.24421) 318 : audit [DBG] from='client.24745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:59.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:56:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:58.966059+0000 mon.smithi121 (mon.0) 825 : audit [DBG] from='client.? 172.21.15.121:0/1725409235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:56:59.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 169561 4.45 MB/sec execute 428 sec latency 36.699 ms 2024-04-23T00:56:59.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:57.954463+0000 mgr.smithi153.olhmgd (mgr.24421) 316 : audit [DBG] from='client.24741 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:59.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:56:58.350662+0000 mgr.smithi153.olhmgd (mgr.24421) 317 : cluster [DBG] pgmap v228: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-04-23T00:56:59.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:58.408469+0000 mgr.smithi153.olhmgd (mgr.24421) 318 : audit [DBG] from='client.24745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:56:59.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:56:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:58.966059+0000 mon.smithi121 (mon.0) 825 : audit [DBG] from='client.? 172.21.15.121:0/1725409235' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:56:59.966 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:57:00.679 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:59.434655+0000 mgr.smithi153.olhmgd (mgr.24421) 319 : audit [DBG] from='client.24751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:00.679 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:56:59.961676+0000 mon.smithi121 (mon.0) 826 : audit [DBG] from='client.? 172.21.15.121:0/1700023003' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:57:00.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 169974 4.48 MB/sec execute 429 sec latency 38.686 ms 2024-04-23T00:57:00.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:59.434655+0000 mgr.smithi153.olhmgd (mgr.24421) 319 : audit [DBG] from='client.24751 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:00.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:56:59.961676+0000 mon.smithi121 (mon.0) 826 : audit [DBG] from='client.? 172.21.15.121:0/1700023003' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:57:01.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 170357 4.51 MB/sec execute 430 sec latency 36.984 ms 2024-04-23T00:57:01.820 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:00.351038+0000 mgr.smithi153.olhmgd (mgr.24421) 320 : cluster [DBG] pgmap v229: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-23T00:57:01.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:00.351038+0000 mgr.smithi153.olhmgd (mgr.24421) 320 : cluster [DBG] pgmap v229: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-23T00:57:02.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 170686 4.52 MB/sec execute 431 sec latency 43.817 ms 2024-04-23T00:57:03.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 171061 4.52 MB/sec execute 432 sec latency 47.318 ms 2024-04-23T00:57:03.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:02.351362+0000 mgr.smithi153.olhmgd (mgr.24421) 321 : cluster [DBG] pgmap v230: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-23T00:57:03.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:02.351362+0000 mgr.smithi153.olhmgd (mgr.24421) 321 : cluster [DBG] pgmap v230: 129 pgs: 129 active+clean; 323 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 703 op/s 2024-04-23T00:57:04.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 171554 4.53 MB/sec execute 433 sec latency 41.113 ms 2024-04-23T00:57:05.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 172410 4.56 MB/sec execute 434 sec latency 29.741 ms 2024-04-23T00:57:05.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:04.352057+0000 mgr.smithi153.olhmgd (mgr.24421) 322 : cluster [DBG] pgmap v231: 129 pgs: 129 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 930 op/s 2024-04-23T00:57:05.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:04.352057+0000 mgr.smithi153.olhmgd (mgr.24421) 322 : cluster [DBG] pgmap v231: 129 pgs: 129 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 930 op/s 2024-04-23T00:57:06.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 173377 4.63 MB/sec execute 435 sec latency 14.313 ms 2024-04-23T00:57:07.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 174030 4.67 MB/sec execute 436 sec latency 33.501 ms 2024-04-23T00:57:07.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:06.352452+0000 mgr.smithi153.olhmgd (mgr.24421) 323 : cluster [DBG] pgmap v232: 129 pgs: 129 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 561 op/s 2024-04-23T00:57:07.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:06.352452+0000 mgr.smithi153.olhmgd (mgr.24421) 323 : cluster [DBG] pgmap v232: 129 pgs: 129 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 561 op/s 2024-04-23T00:57:08.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 174443 4.68 MB/sec execute 437 sec latency 87.462 ms 2024-04-23T00:57:09.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:08.354384+0000 mgr.smithi153.olhmgd (mgr.24421) 324 : cluster [DBG] pgmap v233: 129 pgs: 129 active+clean; 333 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 895 op/s 2024-04-23T00:57:09.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:08.354384+0000 mgr.smithi153.olhmgd (mgr.24421) 324 : cluster [DBG] pgmap v233: 129 pgs: 129 active+clean; 333 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 895 op/s 2024-04-23T00:57:09.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 174766 4.68 MB/sec execute 438 sec latency 53.266 ms 2024-04-23T00:57:10.699 INFO:teuthology.orchestra.run.smithi121.stdout: 5 175202 4.70 MB/sec execute 439 sec latency 38.446 ms 2024-04-23T00:57:11.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 175657 4.70 MB/sec execute 440 sec latency 102.319 ms 2024-04-23T00:57:11.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:10.354835+0000 mgr.smithi153.olhmgd (mgr.24421) 325 : cluster [DBG] pgmap v234: 129 pgs: 129 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-23T00:57:11.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:10.354835+0000 mgr.smithi153.olhmgd (mgr.24421) 325 : cluster [DBG] pgmap v234: 129 pgs: 129 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-23T00:57:12.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:12.402791+0000 mon.smithi153 (mon.1) 155 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:12.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 176141 4.72 MB/sec execute 441 sec latency 41.388 ms 2024-04-23T00:57:12.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:12.402791+0000 mon.smithi153 (mon.1) 155 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:13.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 176604 4.74 MB/sec execute 442 sec latency 29.003 ms 2024-04-23T00:57:13.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:12.355170+0000 mgr.smithi153.olhmgd (mgr.24421) 326 : cluster [DBG] pgmap v235: 129 pgs: 129 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-23T00:57:13.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:12.355170+0000 mgr.smithi153.olhmgd (mgr.24421) 326 : cluster [DBG] pgmap v235: 129 pgs: 129 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 652 op/s 2024-04-23T00:57:14.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 177012 4.77 MB/sec execute 443 sec latency 33.555 ms 2024-04-23T00:57:15.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 177423 4.80 MB/sec execute 444 sec latency 35.898 ms 2024-04-23T00:57:15.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:14.355769+0000 mgr.smithi153.olhmgd (mgr.24421) 327 : cluster [DBG] pgmap v236: 129 pgs: 129 active+clean; 328 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 907 op/s 2024-04-23T00:57:15.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:14.355769+0000 mgr.smithi153.olhmgd (mgr.24421) 327 : cluster [DBG] pgmap v236: 129 pgs: 129 active+clean; 328 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 907 op/s 2024-04-23T00:57:16.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 177780 4.80 MB/sec execute 445 sec latency 60.771 ms 2024-04-23T00:57:17.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 178161 4.80 MB/sec execute 446 sec latency 51.208 ms 2024-04-23T00:57:17.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:16.356158+0000 mgr.smithi153.olhmgd (mgr.24421) 328 : cluster [DBG] pgmap v237: 129 pgs: 129 active+clean; 328 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.4 MiB/s wr, 680 op/s 2024-04-23T00:57:17.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:16.356158+0000 mgr.smithi153.olhmgd (mgr.24421) 328 : cluster [DBG] pgmap v237: 129 pgs: 129 active+clean; 328 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.4 MiB/s wr, 680 op/s 2024-04-23T00:57:18.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 178622 4.82 MB/sec execute 447 sec latency 31.438 ms 2024-04-23T00:57:19.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:18.356793+0000 mgr.smithi153.olhmgd (mgr.24421) 329 : cluster [DBG] pgmap v238: 129 pgs: 129 active+clean; 330 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 938 op/s 2024-04-23T00:57:19.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:18.356793+0000 mgr.smithi153.olhmgd (mgr.24421) 329 : cluster [DBG] pgmap v238: 129 pgs: 129 active+clean; 330 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 938 op/s 2024-04-23T00:57:19.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 179044 4.83 MB/sec execute 448 sec latency 48.174 ms 2024-04-23T00:57:20.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 179512 4.84 MB/sec execute 449 sec latency 46.861 ms 2024-04-23T00:57:21.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 179936 4.85 MB/sec execute 450 sec latency 39.611 ms 2024-04-23T00:57:21.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:20.357169+0000 mgr.smithi153.olhmgd (mgr.24421) 330 : cluster [DBG] pgmap v239: 129 pgs: 129 active+clean; 327 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 607 op/s 2024-04-23T00:57:21.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:20.357169+0000 mgr.smithi153.olhmgd (mgr.24421) 330 : cluster [DBG] pgmap v239: 129 pgs: 129 active+clean; 327 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 607 op/s 2024-04-23T00:57:22.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 180507 4.89 MB/sec execute 451 sec latency 26.455 ms 2024-04-23T00:57:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 181049 4.93 MB/sec execute 452 sec latency 28.980 ms 2024-04-23T00:57:23.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:22.357484+0000 mgr.smithi153.olhmgd (mgr.24421) 331 : cluster [DBG] pgmap v240: 129 pgs: 129 active+clean; 327 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-04-23T00:57:23.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:22.357484+0000 mgr.smithi153.olhmgd (mgr.24421) 331 : cluster [DBG] pgmap v240: 129 pgs: 129 active+clean; 327 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-04-23T00:57:24.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 181406 4.93 MB/sec execute 453 sec latency 50.091 ms 2024-04-23T00:57:25.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 181724 4.93 MB/sec execute 454 sec latency 42.001 ms 2024-04-23T00:57:25.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:24.358161+0000 mgr.smithi153.olhmgd (mgr.24421) 332 : cluster [DBG] pgmap v241: 129 pgs: 129 active+clean; 334 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.9 MiB/s wr, 795 op/s 2024-04-23T00:57:25.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:24.358161+0000 mgr.smithi153.olhmgd (mgr.24421) 332 : cluster [DBG] pgmap v241: 129 pgs: 129 active+clean; 334 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.9 MiB/s wr, 795 op/s 2024-04-23T00:57:26.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 182081 4.93 MB/sec execute 455 sec latency 54.671 ms 2024-04-23T00:57:27.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 182503 4.94 MB/sec execute 456 sec latency 47.034 ms 2024-04-23T00:57:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:26.358505+0000 mgr.smithi153.olhmgd (mgr.24421) 333 : cluster [DBG] pgmap v242: 129 pgs: 129 active+clean; 334 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 539 op/s 2024-04-23T00:57:27.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:27.403057+0000 mon.smithi153 (mon.1) 156 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:26.358505+0000 mgr.smithi153.olhmgd (mgr.24421) 333 : cluster [DBG] pgmap v242: 129 pgs: 129 active+clean; 334 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 539 op/s 2024-04-23T00:57:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:27.403057+0000 mon.smithi153 (mon.1) 156 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:28.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 182927 4.95 MB/sec execute 457 sec latency 43.665 ms 2024-04-23T00:57:29.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:28.359070+0000 mgr.smithi153.olhmgd (mgr.24421) 334 : cluster [DBG] pgmap v243: 129 pgs: 129 active+clean; 324 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 789 op/s 2024-04-23T00:57:29.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:28.359070+0000 mgr.smithi153.olhmgd (mgr.24421) 334 : cluster [DBG] pgmap v243: 129 pgs: 129 active+clean; 324 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 789 op/s 2024-04-23T00:57:29.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 183282 4.96 MB/sec execute 458 sec latency 51.889 ms 2024-04-23T00:57:30.489 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:57:30.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 183981 5.01 MB/sec execute 459 sec latency 36.711 ms 2024-04-23T00:57:31.472 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:57:31.472 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:57:31.472 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (15m) 7m ago 15m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:57:31.472 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (14m) 7m ago 14m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:57:31.472 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (9m) 7m ago 9m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (10m) 7m ago 10m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (17m) 7m ago 17m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (7m) 7m ago 14m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (17m) 7m ago 17m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (14m) 7m ago 14m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (7m) 7m ago 7m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (7m) 7m ago 7m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (14m) 7m ago 14m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (13m) 7m ago 13m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (13m) 7m ago 13m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (13m) 7m ago 13m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (12m) 7m ago 12m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:57:31.473 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (12m) 7m ago 12m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:57:31.474 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (12m) 7m ago 12m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:57:31.474 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (11m) 7m ago 11m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:57:31.474 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (11m) 7m ago 11m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:57:31.474 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (13m) 7m ago 14m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:57:31.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 184542 5.05 MB/sec execute 460 sec latency 25.024 ms 2024-04-23T00:57:31.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:30.359454+0000 mgr.smithi153.olhmgd (mgr.24421) 335 : cluster [DBG] pgmap v244: 129 pgs: 129 active+clean; 323 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 536 op/s 2024-04-23T00:57:31.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:30.469018+0000 mgr.smithi153.olhmgd (mgr.24421) 336 : audit [DBG] from='client.15072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:31.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:30.359454+0000 mgr.smithi153.olhmgd (mgr.24421) 335 : cluster [DBG] pgmap v244: 129 pgs: 129 active+clean; 323 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 536 op/s 2024-04-23T00:57:31.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:30.469018+0000 mgr.smithi153.olhmgd (mgr.24421) 336 : audit [DBG] from='client.15072 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:32.031 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:57:32.031 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:57:32.031 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:57:32.031 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:57:32.032 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:57:32.491 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:57:32.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:30.966256+0000 mgr.smithi153.olhmgd (mgr.24421) 337 : audit [DBG] from='client.15076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:32.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:31.455644+0000 mgr.smithi153.olhmgd (mgr.24421) 338 : audit [DBG] from='client.24763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:32.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:32.027113+0000 mon.smithi153 (mon.1) 157 : audit [DBG] from='client.? 172.21.15.121:0/3448646633' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:57:32.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 185008 5.06 MB/sec execute 461 sec latency 40.220 ms 2024-04-23T00:57:32.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:30.966256+0000 mgr.smithi153.olhmgd (mgr.24421) 337 : audit [DBG] from='client.15076 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:32.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:31.455644+0000 mgr.smithi153.olhmgd (mgr.24421) 338 : audit [DBG] from='client.24763 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:32.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:32.027113+0000 mon.smithi153 (mon.1) 157 : audit [DBG] from='client.? 172.21.15.121:0/3448646633' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:57:32.990 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:57:33.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 185436 5.06 MB/sec execute 462 sec latency 41.573 ms 2024-04-23T00:57:33.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:32.359856+0000 mgr.smithi153.olhmgd (mgr.24421) 339 : cluster [DBG] pgmap v245: 129 pgs: 129 active+clean; 323 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 533 op/s 2024-04-23T00:57:33.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:32.486501+0000 mgr.smithi153.olhmgd (mgr.24421) 340 : audit [DBG] from='client.24771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:33.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:32.986731+0000 mon.smithi121 (mon.0) 827 : audit [DBG] from='client.? 172.21.15.121:0/3190646414' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:57:33.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:32.359856+0000 mgr.smithi153.olhmgd (mgr.24421) 339 : cluster [DBG] pgmap v245: 129 pgs: 129 active+clean; 323 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 533 op/s 2024-04-23T00:57:33.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:32.486501+0000 mgr.smithi153.olhmgd (mgr.24421) 340 : audit [DBG] from='client.24771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:57:33.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:32.986731+0000 mon.smithi121 (mon.0) 827 : audit [DBG] from='client.? 172.21.15.121:0/3190646414' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:57:34.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 186070 5.08 MB/sec execute 463 sec latency 30.403 ms 2024-04-23T00:57:35.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 186761 5.10 MB/sec execute 464 sec latency 23.004 ms 2024-04-23T00:57:35.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:34.360466+0000 mgr.smithi153.olhmgd (mgr.24421) 341 : cluster [DBG] pgmap v246: 129 pgs: 129 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 840 op/s 2024-04-23T00:57:35.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:34.360466+0000 mgr.smithi153.olhmgd (mgr.24421) 341 : cluster [DBG] pgmap v246: 129 pgs: 129 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 840 op/s 2024-04-23T00:57:36.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 187427 5.14 MB/sec execute 465 sec latency 38.738 ms 2024-04-23T00:57:37.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 187996 5.18 MB/sec execute 466 sec latency 41.297 ms 2024-04-23T00:57:37.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:36.360822+0000 mgr.smithi153.olhmgd (mgr.24421) 342 : cluster [DBG] pgmap v247: 129 pgs: 129 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-04-23T00:57:37.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:36.360822+0000 mgr.smithi153.olhmgd (mgr.24421) 342 : cluster [DBG] pgmap v247: 129 pgs: 129 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-04-23T00:57:38.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 188498 5.19 MB/sec execute 467 sec latency 33.366 ms 2024-04-23T00:57:39.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:38.361422+0000 mgr.smithi153.olhmgd (mgr.24421) 343 : cluster [DBG] pgmap v248: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 901 op/s 2024-04-23T00:57:39.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:38.361422+0000 mgr.smithi153.olhmgd (mgr.24421) 343 : cluster [DBG] pgmap v248: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 901 op/s 2024-04-23T00:57:39.700 INFO:teuthology.orchestra.run.smithi121.stdout: 5 188836 5.19 MB/sec execute 468 sec latency 49.130 ms 2024-04-23T00:57:40.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 189625 5.21 MB/sec execute 469 sec latency 27.566 ms 2024-04-23T00:57:41.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 190564 5.26 MB/sec execute 470 sec latency 19.320 ms 2024-04-23T00:57:41.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:40.361817+0000 mgr.smithi153.olhmgd (mgr.24421) 344 : cluster [DBG] pgmap v249: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 667 op/s 2024-04-23T00:57:41.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:40.361817+0000 mgr.smithi153.olhmgd (mgr.24421) 344 : cluster [DBG] pgmap v249: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 667 op/s 2024-04-23T00:57:42.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:42.403566+0000 mon.smithi153 (mon.1) 158 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:42.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 191603 5.33 MB/sec execute 471 sec latency 28.993 ms 2024-04-23T00:57:42.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:42.403566+0000 mon.smithi153 (mon.1) 158 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:43.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 192055 5.34 MB/sec execute 472 sec latency 45.593 ms 2024-04-23T00:57:43.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:42.362158+0000 mgr.smithi153.olhmgd (mgr.24421) 345 : cluster [DBG] pgmap v250: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 662 op/s 2024-04-23T00:57:43.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:42.362158+0000 mgr.smithi153.olhmgd (mgr.24421) 345 : cluster [DBG] pgmap v250: 129 pgs: 129 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 662 op/s 2024-04-23T00:57:44.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 192377 5.34 MB/sec execute 473 sec latency 61.303 ms 2024-04-23T00:57:45.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 192960 5.36 MB/sec execute 474 sec latency 34.574 ms 2024-04-23T00:57:45.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:44.362915+0000 mgr.smithi153.olhmgd (mgr.24421) 346 : cluster [DBG] pgmap v251: 129 pgs: 129 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-23T00:57:45.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:44.362915+0000 mgr.smithi153.olhmgd (mgr.24421) 346 : cluster [DBG] pgmap v251: 129 pgs: 129 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-23T00:57:46.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 193836 5.39 MB/sec execute 475 sec latency 43.910 ms 2024-04-23T00:57:47.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 194470 5.42 MB/sec execute 476 sec latency 23.801 ms 2024-04-23T00:57:47.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:46.363306+0000 mgr.smithi153.olhmgd (mgr.24421) 347 : cluster [DBG] pgmap v252: 129 pgs: 129 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.1 MiB/s wr, 777 op/s 2024-04-23T00:57:47.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:46.363306+0000 mgr.smithi153.olhmgd (mgr.24421) 347 : cluster [DBG] pgmap v252: 129 pgs: 129 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.1 MiB/s wr, 777 op/s 2024-04-23T00:57:48.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 194954 5.45 MB/sec execute 477 sec latency 44.803 ms 2024-04-23T00:57:49.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:48.363878+0000 mgr.smithi153.olhmgd (mgr.24421) 348 : cluster [DBG] pgmap v253: 129 pgs: 129 active+clean; 341 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-23T00:57:49.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:48.363878+0000 mgr.smithi153.olhmgd (mgr.24421) 348 : cluster [DBG] pgmap v253: 129 pgs: 129 active+clean; 341 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-23T00:57:49.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 195346 5.46 MB/sec execute 478 sec latency 43.037 ms 2024-04-23T00:57:50.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 195902 5.47 MB/sec execute 479 sec latency 46.429 ms 2024-04-23T00:57:51.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 196576 5.49 MB/sec execute 480 sec latency 44.015 ms 2024-04-23T00:57:51.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:50.365528+0000 mgr.smithi153.olhmgd (mgr.24421) 349 : cluster [DBG] pgmap v254: 129 pgs: 129 active+clean; 345 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-04-23T00:57:51.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:50.365528+0000 mgr.smithi153.olhmgd (mgr.24421) 349 : cluster [DBG] pgmap v254: 129 pgs: 129 active+clean; 345 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 778 op/s 2024-04-23T00:57:52.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 197306 5.52 MB/sec execute 481 sec latency 30.305 ms 2024-04-23T00:57:53.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 197967 5.55 MB/sec execute 482 sec latency 24.936 ms 2024-04-23T00:57:53.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:52.365852+0000 mgr.smithi153.olhmgd (mgr.24421) 350 : cluster [DBG] pgmap v255: 129 pgs: 129 active+clean; 345 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 761 op/s 2024-04-23T00:57:53.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:52.365852+0000 mgr.smithi153.olhmgd (mgr.24421) 350 : cluster [DBG] pgmap v255: 129 pgs: 129 active+clean; 345 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 761 op/s 2024-04-23T00:57:54.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 198584 5.59 MB/sec execute 483 sec latency 37.656 ms 2024-04-23T00:57:55.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 198954 5.59 MB/sec execute 484 sec latency 53.628 ms 2024-04-23T00:57:55.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:54.366560+0000 mgr.smithi153.olhmgd (mgr.24421) 351 : cluster [DBG] pgmap v256: 129 pgs: 129 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.12k op/s 2024-04-23T00:57:55.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:54.366560+0000 mgr.smithi153.olhmgd (mgr.24421) 351 : cluster [DBG] pgmap v256: 129 pgs: 129 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.12k op/s 2024-04-23T00:57:56.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 199251 5.60 MB/sec execute 485 sec latency 55.466 ms 2024-04-23T00:57:57.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 199559 5.60 MB/sec execute 486 sec latency 43.652 ms 2024-04-23T00:57:57.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:56.367063+0000 mgr.smithi153.olhmgd (mgr.24421) 352 : cluster [DBG] pgmap v257: 129 pgs: 129 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 697 op/s 2024-04-23T00:57:57.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:57:57.403647+0000 mon.smithi153 (mon.1) 159 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:57.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:56.367063+0000 mgr.smithi153.olhmgd (mgr.24421) 352 : cluster [DBG] pgmap v257: 129 pgs: 129 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 697 op/s 2024-04-23T00:57:57.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:57:57.403647+0000 mon.smithi153 (mon.1) 159 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:57:58.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 199944 5.60 MB/sec execute 487 sec latency 46.972 ms 2024-04-23T00:57:59.576 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:57:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:57:58.367642+0000 mgr.smithi153.olhmgd (mgr.24421) 353 : cluster [DBG] pgmap v258: 129 pgs: 129 active+clean; 326 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 963 op/s 2024-04-23T00:57:59.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:57:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:57:58.367642+0000 mgr.smithi153.olhmgd (mgr.24421) 353 : cluster [DBG] pgmap v258: 129 pgs: 129 active+clean; 326 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 963 op/s 2024-04-23T00:57:59.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 200314 5.61 MB/sec execute 488 sec latency 49.345 ms 2024-04-23T00:58:00.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 200721 5.61 MB/sec execute 489 sec latency 43.570 ms 2024-04-23T00:58:01.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 201294 5.63 MB/sec execute 490 sec latency 37.188 ms 2024-04-23T00:58:01.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:00.368082+0000 mgr.smithi153.olhmgd (mgr.24421) 354 : cluster [DBG] pgmap v259: 129 pgs: 129 active+clean; 325 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 736 op/s 2024-04-23T00:58:01.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:00.368082+0000 mgr.smithi153.olhmgd (mgr.24421) 354 : cluster [DBG] pgmap v259: 129 pgs: 129 active+clean; 325 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 736 op/s 2024-04-23T00:58:02.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 201880 5.67 MB/sec execute 491 sec latency 26.545 ms 2024-04-23T00:58:03.442 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:58:03.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 202321 5.69 MB/sec execute 492 sec latency 51.762 ms 2024-04-23T00:58:03.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:02.368415+0000 mgr.smithi153.olhmgd (mgr.24421) 355 : cluster [DBG] pgmap v260: 129 pgs: 129 active+clean; 325 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 627 op/s 2024-04-23T00:58:03.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:02.368415+0000 mgr.smithi153.olhmgd (mgr.24421) 355 : cluster [DBG] pgmap v260: 129 pgs: 129 active+clean; 325 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 627 op/s 2024-04-23T00:58:04.162 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (16m) 7m ago 16m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (15m) 7m ago 15m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (10m) 7m ago 10m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (10m) 7m ago 10m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (17m) 7m ago 17m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (8m) 7m ago 15m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (17m) 7m ago 17m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (15m) 7m ago 15m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (7m) 7m ago 7m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:58:04.163 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (7m) 7m ago 7m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (14m) 7m ago 14m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (14m) 7m ago 14m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (14m) 7m ago 14m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (13m) 7m ago 13m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (13m) 7m ago 13m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (13m) 7m ago 13m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (12m) 7m ago 12m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (12m) 7m ago 12m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (12m) 7m ago 12m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:58:04.164 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (14m) 7m ago 15m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:58:04.634 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:58:04.635 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:58:04.701 INFO:teuthology.orchestra.run.smithi121.stdout: 5 202615 5.69 MB/sec execute 493 sec latency 55.358 ms 2024-04-23T00:58:04.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:03.423665+0000 mgr.smithi153.olhmgd (mgr.24421) 356 : audit [DBG] from='client.15092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:04.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:03.794224+0000 mgr.smithi153.olhmgd (mgr.24421) 357 : audit [DBG] from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:04.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:03.423665+0000 mgr.smithi153.olhmgd (mgr.24421) 356 : audit [DBG] from='client.15092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:04.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:03.794224+0000 mgr.smithi153.olhmgd (mgr.24421) 357 : audit [DBG] from='client.15096 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:04.995 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:58:04.995 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:58:04.995 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:58:04.995 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:58:04.996 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:58:04.996 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:58:04.996 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:58:04.996 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:58:04.996 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:58:05.463 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:58:05.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 202920 5.69 MB/sec execute 494 sec latency 68.969 ms 2024-04-23T00:58:05.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:04.146342+0000 mgr.smithi153.olhmgd (mgr.24421) 358 : audit [DBG] from='client.15100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:05.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:04.369136+0000 mgr.smithi153.olhmgd (mgr.24421) 359 : cluster [DBG] pgmap v261: 129 pgs: 129 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 902 op/s 2024-04-23T00:58:05.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:04.629274+0000 mon.smithi153 (mon.1) 160 : audit [DBG] from='client.? 172.21.15.121:0/1803169840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:58:05.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:04.146342+0000 mgr.smithi153.olhmgd (mgr.24421) 358 : audit [DBG] from='client.15100 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:05.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:04.369136+0000 mgr.smithi153.olhmgd (mgr.24421) 359 : cluster [DBG] pgmap v261: 129 pgs: 129 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 902 op/s 2024-04-23T00:58:05.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:04.629274+0000 mon.smithi153 (mon.1) 160 : audit [DBG] from='client.? 172.21.15.121:0/1803169840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:58:06.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 203403 5.70 MB/sec execute 495 sec latency 63.164 ms 2024-04-23T00:58:06.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:04.990566+0000 mgr.smithi153.olhmgd (mgr.24421) 360 : audit [DBG] from='client.15108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:06.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:05.459128+0000 mon.smithi121 (mon.0) 828 : audit [DBG] from='client.? 172.21.15.121:0/791095517' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:58:06.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:04.990566+0000 mgr.smithi153.olhmgd (mgr.24421) 360 : audit [DBG] from='client.15108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:06.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:05.459128+0000 mon.smithi121 (mon.0) 828 : audit [DBG] from='client.? 172.21.15.121:0/791095517' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:58:07.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 203894 5.71 MB/sec execute 496 sec latency 34.299 ms 2024-04-23T00:58:07.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:06.369459+0000 mgr.smithi153.olhmgd (mgr.24421) 361 : cluster [DBG] pgmap v262: 129 pgs: 129 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 544 op/s 2024-04-23T00:58:07.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:06.369459+0000 mgr.smithi153.olhmgd (mgr.24421) 361 : cluster [DBG] pgmap v262: 129 pgs: 129 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 544 op/s 2024-04-23T00:58:08.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 204549 5.73 MB/sec execute 497 sec latency 35.810 ms 2024-04-23T00:58:09.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:08.370052+0000 mgr.smithi153.olhmgd (mgr.24421) 362 : cluster [DBG] pgmap v263: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 810 op/s 2024-04-23T00:58:09.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:08.370052+0000 mgr.smithi153.olhmgd (mgr.24421) 362 : cluster [DBG] pgmap v263: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 810 op/s 2024-04-23T00:58:09.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 205297 5.77 MB/sec execute 498 sec latency 25.975 ms 2024-04-23T00:58:10.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 205822 5.80 MB/sec execute 499 sec latency 40.319 ms 2024-04-23T00:58:11.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 206249 5.81 MB/sec execute 500 sec latency 55.819 ms 2024-04-23T00:58:11.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:10.370449+0000 mgr.smithi153.olhmgd (mgr.24421) 363 : cluster [DBG] pgmap v264: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 550 op/s 2024-04-23T00:58:11.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:10.370449+0000 mgr.smithi153.olhmgd (mgr.24421) 363 : cluster [DBG] pgmap v264: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 550 op/s 2024-04-23T00:58:12.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:12.403933+0000 mon.smithi153 (mon.1) 161 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:12.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 206615 5.81 MB/sec execute 501 sec latency 49.045 ms 2024-04-23T00:58:12.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:12.403933+0000 mon.smithi153 (mon.1) 161 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:13.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 207061 5.81 MB/sec execute 502 sec latency 72.616 ms 2024-04-23T00:58:13.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:12.370839+0000 mgr.smithi153.olhmgd (mgr.24421) 364 : cluster [DBG] pgmap v265: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 547 op/s 2024-04-23T00:58:13.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:12.370839+0000 mgr.smithi153.olhmgd (mgr.24421) 364 : cluster [DBG] pgmap v265: 129 pgs: 129 active+clean; 323 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 547 op/s 2024-04-23T00:58:14.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 207565 5.82 MB/sec execute 503 sec latency 39.258 ms 2024-04-23T00:58:15.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 208125 5.84 MB/sec execute 504 sec latency 27.797 ms 2024-04-23T00:58:15.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:14.371613+0000 mgr.smithi153.olhmgd (mgr.24421) 365 : cluster [DBG] pgmap v266: 129 pgs: 129 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 874 op/s 2024-04-23T00:58:15.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:14.371613+0000 mgr.smithi153.olhmgd (mgr.24421) 365 : cluster [DBG] pgmap v266: 129 pgs: 129 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 874 op/s 2024-04-23T00:58:16.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 208751 5.87 MB/sec execute 505 sec latency 27.739 ms 2024-04-23T00:58:17.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 209198 5.90 MB/sec execute 506 sec latency 37.604 ms 2024-04-23T00:58:17.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:16.372004+0000 mgr.smithi153.olhmgd (mgr.24421) 366 : cluster [DBG] pgmap v267: 129 pgs: 129 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-04-23T00:58:17.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:16.372004+0000 mgr.smithi153.olhmgd (mgr.24421) 366 : cluster [DBG] pgmap v267: 129 pgs: 129 active+clean; 332 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-04-23T00:58:18.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 209530 5.90 MB/sec execute 507 sec latency 38.127 ms 2024-04-23T00:58:19.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:18.372737+0000 mgr.smithi153.olhmgd (mgr.24421) 367 : cluster [DBG] pgmap v268: 129 pgs: 129 active+clean; 333 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 851 op/s 2024-04-23T00:58:19.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:18.372737+0000 mgr.smithi153.olhmgd (mgr.24421) 367 : cluster [DBG] pgmap v268: 129 pgs: 129 active+clean; 333 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 851 op/s 2024-04-23T00:58:19.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 209853 5.90 MB/sec execute 508 sec latency 58.996 ms 2024-04-23T00:58:20.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 210506 5.92 MB/sec execute 509 sec latency 50.051 ms 2024-04-23T00:58:21.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 211342 5.94 MB/sec execute 510 sec latency 34.124 ms 2024-04-23T00:58:21.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:20.373254+0000 mgr.smithi153.olhmgd (mgr.24421) 368 : cluster [DBG] pgmap v269: 129 pgs: 129 active+clean; 335 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 629 op/s 2024-04-23T00:58:21.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:20.373254+0000 mgr.smithi153.olhmgd (mgr.24421) 368 : cluster [DBG] pgmap v269: 129 pgs: 129 active+clean; 335 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 629 op/s 2024-04-23T00:58:22.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 212343 5.99 MB/sec execute 511 sec latency 16.714 ms 2024-04-23T00:58:23.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 212858 6.02 MB/sec execute 512 sec latency 63.948 ms 2024-04-23T00:58:23.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:22.373653+0000 mgr.smithi153.olhmgd (mgr.24421) 369 : cluster [DBG] pgmap v270: 129 pgs: 129 active+clean; 335 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 623 op/s 2024-04-23T00:58:23.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:22.373653+0000 mgr.smithi153.olhmgd (mgr.24421) 369 : cluster [DBG] pgmap v270: 129 pgs: 129 active+clean; 335 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 623 op/s 2024-04-23T00:58:24.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 213283 6.03 MB/sec execute 513 sec latency 51.379 ms 2024-04-23T00:58:25.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 213655 6.03 MB/sec execute 514 sec latency 44.254 ms 2024-04-23T00:58:25.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:24.374378+0000 mgr.smithi153.olhmgd (mgr.24421) 370 : cluster [DBG] pgmap v271: 129 pgs: 129 active+clean; 337 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-23T00:58:25.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:24.374378+0000 mgr.smithi153.olhmgd (mgr.24421) 370 : cluster [DBG] pgmap v271: 129 pgs: 129 active+clean; 337 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-23T00:58:26.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 214138 6.04 MB/sec execute 515 sec latency 58.831 ms 2024-04-23T00:58:27.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 214699 6.05 MB/sec execute 516 sec latency 46.846 ms 2024-04-23T00:58:27.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:26.374781+0000 mgr.smithi153.olhmgd (mgr.24421) 371 : cluster [DBG] pgmap v272: 129 pgs: 129 active+clean; 337 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 691 op/s 2024-04-23T00:58:27.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:27.404218+0000 mon.smithi153 (mon.1) 162 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:26.374781+0000 mgr.smithi153.olhmgd (mgr.24421) 371 : cluster [DBG] pgmap v272: 129 pgs: 129 active+clean; 337 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 691 op/s 2024-04-23T00:58:27.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:27.404218+0000 mon.smithi153 (mon.1) 162 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:28.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 215266 6.07 MB/sec execute 517 sec latency 24.321 ms 2024-04-23T00:58:29.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:28.375485+0000 mgr.smithi153.olhmgd (mgr.24421) 372 : cluster [DBG] pgmap v273: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 986 op/s 2024-04-23T00:58:29.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:28.375485+0000 mgr.smithi153.olhmgd (mgr.24421) 372 : cluster [DBG] pgmap v273: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 986 op/s 2024-04-23T00:58:29.702 INFO:teuthology.orchestra.run.smithi121.stdout: 5 215826 6.09 MB/sec execute 518 sec latency 33.785 ms 2024-04-23T00:58:30.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 216259 6.11 MB/sec execute 519 sec latency 46.287 ms 2024-04-23T00:58:31.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 216636 6.12 MB/sec execute 520 sec latency 53.756 ms 2024-04-23T00:58:31.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:30.375889+0000 mgr.smithi153.olhmgd (mgr.24421) 373 : cluster [DBG] pgmap v274: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 733 op/s 2024-04-23T00:58:31.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:30.375889+0000 mgr.smithi153.olhmgd (mgr.24421) 373 : cluster [DBG] pgmap v274: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 733 op/s 2024-04-23T00:58:32.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 216920 6.12 MB/sec execute 521 sec latency 68.526 ms 2024-04-23T00:58:33.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 217182 6.12 MB/sec execute 522 sec latency 44.980 ms 2024-04-23T00:58:33.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:32.376358+0000 mgr.smithi153.olhmgd (mgr.24421) 374 : cluster [DBG] pgmap v275: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 690 op/s 2024-04-23T00:58:33.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:32.376358+0000 mgr.smithi153.olhmgd (mgr.24421) 374 : cluster [DBG] pgmap v275: 129 pgs: 129 active+clean; 321 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 690 op/s 2024-04-23T00:58:34.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 217423 6.12 MB/sec execute 523 sec latency 55.016 ms 2024-04-23T00:58:35.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 217733 6.12 MB/sec execute 524 sec latency 60.977 ms 2024-04-23T00:58:35.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:34.378515+0000 mgr.smithi153.olhmgd (mgr.24421) 375 : cluster [DBG] pgmap v276: 129 pgs: 129 active+clean; 327 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 934 op/s 2024-04-23T00:58:35.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:34.378515+0000 mgr.smithi153.olhmgd (mgr.24421) 375 : cluster [DBG] pgmap v276: 129 pgs: 129 active+clean; 327 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 934 op/s 2024-04-23T00:58:35.932 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:58:36.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 218194 6.13 MB/sec execute 525 sec latency 44.882 ms 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (16m) 8m ago 16m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (15m) 8m ago 15m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:58:36.840 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (11m) 8m ago 11m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (11m) 8m ago 11m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (18m) 8m ago 18m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (9m) 8m ago 15m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (18m) 8m ago 18m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (15m) 8m ago 15m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (8m) 8m ago 8m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (8m) 8m ago 8m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (15m) 8m ago 15m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (15m) 8m ago 15m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (14m) 8m ago 14m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (14m) 8m ago 14m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (13m) 8m ago 13m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (13m) 8m ago 13m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (13m) 8m ago 13m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:58:36.841 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (12m) 8m ago 12m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:58:36.842 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (12m) 8m ago 12m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:58:36.842 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (14m) 8m ago 15m 40.2M - 2.18.1 de242295e225 33d3cc45332e 2024-04-23T00:58:37.389 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:58:37.390 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:58:37.391 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:58:37.391 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:58:37.391 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:58:37.391 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:58:37.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 218587 6.13 MB/sec execute 526 sec latency 40.090 ms 2024-04-23T00:58:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:35.910620+0000 mgr.smithi153.olhmgd (mgr.24421) 376 : audit [DBG] from='client.15116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:36.369528+0000 mgr.smithi153.olhmgd (mgr.24421) 377 : audit [DBG] from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:36.378820+0000 mgr.smithi153.olhmgd (mgr.24421) 378 : cluster [DBG] pgmap v277: 129 pgs: 129 active+clean; 327 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-04-23T00:58:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:36.824659+0000 mgr.smithi153.olhmgd (mgr.24421) 379 : audit [DBG] from='client.15124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:37.384743+0000 mon.smithi153 (mon.1) 163 : audit [DBG] from='client.? 172.21.15.121:0/3639272847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:58:37.855 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:58:37.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:35.910620+0000 mgr.smithi153.olhmgd (mgr.24421) 376 : audit [DBG] from='client.15116 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:36.369528+0000 mgr.smithi153.olhmgd (mgr.24421) 377 : audit [DBG] from='client.15120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:36.378820+0000 mgr.smithi153.olhmgd (mgr.24421) 378 : cluster [DBG] pgmap v277: 129 pgs: 129 active+clean; 327 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-04-23T00:58:37.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:36.824659+0000 mgr.smithi153.olhmgd (mgr.24421) 379 : audit [DBG] from='client.15124 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:37.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:37.384743+0000 mon.smithi153 (mon.1) 163 : audit [DBG] from='client.? 172.21.15.121:0/3639272847' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:58:38.411 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:58:38.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 219040 6.14 MB/sec execute 527 sec latency 51.328 ms 2024-04-23T00:58:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:37.849827+0000 mgr.smithi153.olhmgd (mgr.24421) 380 : audit [DBG] from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:38.406648+0000 mon.smithi121 (mon.0) 829 : audit [DBG] from='client.? 172.21.15.121:0/746662077' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:58:38.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:37.849827+0000 mgr.smithi153.olhmgd (mgr.24421) 380 : audit [DBG] from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:58:38.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:38.406648+0000 mon.smithi121 (mon.0) 829 : audit [DBG] from='client.? 172.21.15.121:0/746662077' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:58:39.703 INFO:teuthology.orchestra.run.smithi121.stdout: 5 219552 6.17 MB/sec execute 528 sec latency 48.543 ms 2024-04-23T00:58:39.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:38.379526+0000 mgr.smithi153.olhmgd (mgr.24421) 381 : cluster [DBG] pgmap v278: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 766 op/s 2024-04-23T00:58:39.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:38.379526+0000 mgr.smithi153.olhmgd (mgr.24421) 381 : cluster [DBG] pgmap v278: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 766 op/s 2024-04-23T00:58:40.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 220025 6.19 MB/sec execute 529 sec latency 34.167 ms 2024-04-23T00:58:41.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 220447 6.20 MB/sec execute 530 sec latency 44.104 ms 2024-04-23T00:58:41.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:40.379870+0000 mgr.smithi153.olhmgd (mgr.24421) 382 : cluster [DBG] pgmap v279: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-04-23T00:58:41.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:40.379870+0000 mgr.smithi153.olhmgd (mgr.24421) 382 : cluster [DBG] pgmap v279: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-04-23T00:58:42.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:42.404691+0000 mon.smithi153 (mon.1) 164 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:42.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 220781 6.20 MB/sec execute 531 sec latency 55.950 ms 2024-04-23T00:58:42.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:42.404691+0000 mon.smithi153 (mon.1) 164 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:43.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 221180 6.20 MB/sec execute 532 sec latency 45.903 ms 2024-04-23T00:58:43.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:42.380712+0000 mgr.smithi153.olhmgd (mgr.24421) 383 : cluster [DBG] pgmap v280: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-04-23T00:58:43.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:42.380712+0000 mgr.smithi153.olhmgd (mgr.24421) 383 : cluster [DBG] pgmap v280: 129 pgs: 129 active+clean; 324 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-04-23T00:58:44.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 221640 6.21 MB/sec execute 533 sec latency 42.124 ms 2024-04-23T00:58:45.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 222001 6.21 MB/sec execute 534 sec latency 57.591 ms 2024-04-23T00:58:45.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:44.381452+0000 mgr.smithi153.olhmgd (mgr.24421) 384 : cluster [DBG] pgmap v281: 129 pgs: 129 active+clean; 327 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 747 op/s 2024-04-23T00:58:45.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:44.381452+0000 mgr.smithi153.olhmgd (mgr.24421) 384 : cluster [DBG] pgmap v281: 129 pgs: 129 active+clean; 327 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 747 op/s 2024-04-23T00:58:46.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 222473 6.23 MB/sec execute 535 sec latency 61.989 ms 2024-04-23T00:58:47.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 223078 6.26 MB/sec execute 536 sec latency 39.420 ms 2024-04-23T00:58:47.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:46.381781+0000 mgr.smithi153.olhmgd (mgr.24421) 385 : cluster [DBG] pgmap v282: 129 pgs: 129 active+clean; 327 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-23T00:58:47.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:46.381781+0000 mgr.smithi153.olhmgd (mgr.24421) 385 : cluster [DBG] pgmap v282: 129 pgs: 129 active+clean; 327 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-23T00:58:48.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 223550 6.28 MB/sec execute 537 sec latency 37.348 ms 2024-04-23T00:58:49.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:48.382343+0000 mgr.smithi153.olhmgd (mgr.24421) 386 : cluster [DBG] pgmap v283: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 776 op/s 2024-04-23T00:58:49.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:48.382343+0000 mgr.smithi153.olhmgd (mgr.24421) 386 : cluster [DBG] pgmap v283: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 776 op/s 2024-04-23T00:58:49.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 223987 6.29 MB/sec execute 538 sec latency 35.867 ms 2024-04-23T00:58:50.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 224304 6.28 MB/sec execute 539 sec latency 44.929 ms 2024-04-23T00:58:51.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 224738 6.29 MB/sec execute 540 sec latency 51.357 ms 2024-04-23T00:58:51.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:50.384376+0000 mgr.smithi153.olhmgd (mgr.24421) 387 : cluster [DBG] pgmap v284: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 549 op/s 2024-04-23T00:58:51.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:50.384376+0000 mgr.smithi153.olhmgd (mgr.24421) 387 : cluster [DBG] pgmap v284: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 549 op/s 2024-04-23T00:58:52.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 225269 6.30 MB/sec execute 541 sec latency 48.073 ms 2024-04-23T00:58:53.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 225737 6.30 MB/sec execute 542 sec latency 42.626 ms 2024-04-23T00:58:53.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:52.384735+0000 mgr.smithi153.olhmgd (mgr.24421) 388 : cluster [DBG] pgmap v285: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 549 op/s 2024-04-23T00:58:53.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:52.384735+0000 mgr.smithi153.olhmgd (mgr.24421) 388 : cluster [DBG] pgmap v285: 129 pgs: 129 active+clean; 330 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 549 op/s 2024-04-23T00:58:54.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 226177 6.31 MB/sec execute 543 sec latency 40.842 ms 2024-04-23T00:58:55.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 226638 6.34 MB/sec execute 544 sec latency 33.669 ms 2024-04-23T00:58:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:54.385313+0000 mgr.smithi153.olhmgd (mgr.24421) 389 : cluster [DBG] pgmap v286: 129 pgs: 129 active+clean; 322 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 832 op/s 2024-04-23T00:58:55.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:54.385313+0000 mgr.smithi153.olhmgd (mgr.24421) 389 : cluster [DBG] pgmap v286: 129 pgs: 129 active+clean; 322 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 832 op/s 2024-04-23T00:58:56.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 227065 6.36 MB/sec execute 545 sec latency 42.711 ms 2024-04-23T00:58:57.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 227584 6.37 MB/sec execute 546 sec latency 21.656 ms 2024-04-23T00:58:57.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:56.385697+0000 mgr.smithi153.olhmgd (mgr.24421) 390 : cluster [DBG] pgmap v287: 129 pgs: 129 active+clean; 322 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 555 op/s 2024-04-23T00:58:57.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:58:57.404851+0000 mon.smithi153 (mon.1) 165 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:57.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:56.385697+0000 mgr.smithi153.olhmgd (mgr.24421) 390 : cluster [DBG] pgmap v287: 129 pgs: 129 active+clean; 322 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 555 op/s 2024-04-23T00:58:57.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:58:57.404851+0000 mon.smithi153 (mon.1) 165 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:58:58.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 227953 6.37 MB/sec execute 547 sec latency 47.947 ms 2024-04-23T00:58:59.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:58:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:58:58.386440+0000 mgr.smithi153.olhmgd (mgr.24421) 391 : cluster [DBG] pgmap v288: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 837 op/s 2024-04-23T00:58:59.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:58:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:58:58.386440+0000 mgr.smithi153.olhmgd (mgr.24421) 391 : cluster [DBG] pgmap v288: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 837 op/s 2024-04-23T00:58:59.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 228398 6.37 MB/sec execute 548 sec latency 40.748 ms 2024-04-23T00:59:00.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 228914 6.38 MB/sec execute 549 sec latency 63.492 ms 2024-04-23T00:59:01.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 229490 6.40 MB/sec execute 550 sec latency 28.939 ms 2024-04-23T00:59:01.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:00.386837+0000 mgr.smithi153.olhmgd (mgr.24421) 392 : cluster [DBG] pgmap v289: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-04-23T00:59:01.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:00.386837+0000 mgr.smithi153.olhmgd (mgr.24421) 392 : cluster [DBG] pgmap v289: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-04-23T00:59:02.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 230034 6.42 MB/sec execute 551 sec latency 33.156 ms 2024-04-23T00:59:03.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:02.387217+0000 mgr.smithi153.olhmgd (mgr.24421) 393 : cluster [DBG] pgmap v290: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-04-23T00:59:03.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 230475 6.44 MB/sec execute 552 sec latency 40.794 ms 2024-04-23T00:59:03.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:02.387217+0000 mgr.smithi153.olhmgd (mgr.24421) 393 : cluster [DBG] pgmap v290: 129 pgs: 129 active+clean; 336 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.7 MiB/s wr, 564 op/s 2024-04-23T00:59:04.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 230860 6.45 MB/sec execute 553 sec latency 51.623 ms 2024-04-23T00:59:05.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:03.737917+0000 mon.smithi121 (mon.0) 830 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:59:05.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:03.743778+0000 mon.smithi121 (mon.0) 831 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:59:05.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:59:03.793399+0000 mgr.smithi153.olhmgd (mgr.24421) 394 : cephadm [INF] Reconfiguring node-exporter.smithi153 (dependencies changed)... 2024-04-23T00:59:05.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T00:59:03.793765+0000 mgr.smithi153.olhmgd (mgr.24421) 395 : cephadm [INF] Reconfiguring daemon node-exporter.smithi153 on smithi153 2024-04-23T00:59:05.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:03.737917+0000 mon.smithi121 (mon.0) 830 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:59:05.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:03.743778+0000 mon.smithi121 (mon.0) 831 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T00:59:05.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:59:03.793399+0000 mgr.smithi153.olhmgd (mgr.24421) 394 : cephadm [INF] Reconfiguring node-exporter.smithi153 (dependencies changed)... 2024-04-23T00:59:05.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:04 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T00:59:03.793765+0000 mgr.smithi153.olhmgd (mgr.24421) 395 : cephadm [INF] Reconfiguring daemon node-exporter.smithi153 on smithi153 2024-04-23T00:59:05.704 INFO:teuthology.orchestra.run.smithi121.stdout: 5 231196 6.45 MB/sec execute 554 sec latency 51.575 ms 2024-04-23T00:59:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:04.387931+0000 mgr.smithi153.olhmgd (mgr.24421) 396 : cluster [DBG] pgmap v291: 129 pgs: 129 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 860 op/s 2024-04-23T00:59:06.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:04.387931+0000 mgr.smithi153.olhmgd (mgr.24421) 396 : cluster [DBG] pgmap v291: 129 pgs: 129 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 860 op/s 2024-04-23T00:59:06.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 231456 6.45 MB/sec execute 555 sec latency 52.170 ms 2024-04-23T00:59:07.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 231758 6.45 MB/sec execute 556 sec latency 62.932 ms 2024-04-23T00:59:08.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:06.388326+0000 mgr.smithi153.olhmgd (mgr.24421) 397 : cluster [DBG] pgmap v292: 129 pgs: 129 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 577 op/s 2024-04-23T00:59:08.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:06.388326+0000 mgr.smithi153.olhmgd (mgr.24421) 397 : cluster [DBG] pgmap v292: 129 pgs: 129 active+clean; 331 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 577 op/s 2024-04-23T00:59:08.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 232109 6.44 MB/sec execute 557 sec latency 76.456 ms 2024-04-23T00:59:08.791 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:59:09.542 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (17m) 8m ago 17m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (16m) 8m ago 16m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (11m) 8m ago 11m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:59:09.543 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (11m) 8m ago 11m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (18m) 8m ago 18m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (9m) 8m ago 16m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (18m) 8m ago 18m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (16m) 8m ago 16m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (9m) 8m ago 9m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:59:09.544 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (8m) 8m ago 8m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (15m) 8m ago 15m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (15m) 8m ago 15m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (15m) 8m ago 15m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (14m) 8m ago 14m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:59:09.545 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (14m) 8m ago 14m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:59:09.546 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (14m) 8m ago 14m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:59:09.546 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (13m) 8m ago 13m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:59:09.546 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (13m) 8m ago 13m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:59:09.546 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (13m) 8m ago 13m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:59:09.546 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 starting - - - - 2024-04-23T00:59:09.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:08.389114+0000 mgr.smithi153.olhmgd (mgr.24421) 398 : cluster [DBG] pgmap v293: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-04-23T00:59:09.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:08.772784+0000 mgr.smithi153.olhmgd (mgr.24421) 399 : audit [DBG] from='client.15140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:09.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:08.389114+0000 mgr.smithi153.olhmgd (mgr.24421) 398 : cluster [DBG] pgmap v293: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-04-23T00:59:09.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:08.772784+0000 mgr.smithi153.olhmgd (mgr.24421) 399 : audit [DBG] from='client.15140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:09.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 232521 6.45 MB/sec execute 558 sec latency 50.818 ms 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:59:10.075 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:59:10.076 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:59:10.532 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:59:10.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:09.142922+0000 mgr.smithi153.olhmgd (mgr.24421) 400 : audit [DBG] from='client.24825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:10.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:09.525972+0000 mgr.smithi153.olhmgd (mgr.24421) 401 : audit [DBG] from='client.15146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:10.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:10.069927+0000 mon.smithi153 (mon.1) 166 : audit [DBG] from='client.? 172.21.15.121:0/2579723004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:59:10.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:09.142922+0000 mgr.smithi153.olhmgd (mgr.24421) 400 : audit [DBG] from='client.24825 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:10.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:09.525972+0000 mgr.smithi153.olhmgd (mgr.24421) 401 : audit [DBG] from='client.15146 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:10.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:10.069927+0000 mon.smithi153 (mon.1) 166 : audit [DBG] from='client.? 172.21.15.121:0/2579723004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:59:10.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 233075 6.47 MB/sec execute 559 sec latency 29.815 ms 2024-04-23T00:59:11.062 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:59:11.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:10.389736+0000 mgr.smithi153.olhmgd (mgr.24421) 402 : cluster [DBG] pgmap v294: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 519 op/s 2024-04-23T00:59:11.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:10.526524+0000 mgr.smithi153.olhmgd (mgr.24421) 403 : audit [DBG] from='client.15152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:11.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:11.057096+0000 mon.smithi121 (mon.0) 832 : audit [DBG] from='client.? 172.21.15.121:0/3359877477' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:59:11.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 233697 6.50 MB/sec execute 560 sec latency 33.997 ms 2024-04-23T00:59:11.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:10.389736+0000 mgr.smithi153.olhmgd (mgr.24421) 402 : cluster [DBG] pgmap v294: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 519 op/s 2024-04-23T00:59:11.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:10.526524+0000 mgr.smithi153.olhmgd (mgr.24421) 403 : audit [DBG] from='client.15152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:11.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:11.057096+0000 mon.smithi121 (mon.0) 832 : audit [DBG] from='client.? 172.21.15.121:0/3359877477' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:59:12.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 234261 6.52 MB/sec execute 561 sec latency 27.913 ms 2024-04-23T00:59:12.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:12.406540+0000 mon.smithi153 (mon.1) 167 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:12.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:12.406540+0000 mon.smithi153 (mon.1) 167 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:13.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 234728 6.53 MB/sec execute 562 sec latency 62.042 ms 2024-04-23T00:59:13.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:12.390091+0000 mgr.smithi153.olhmgd (mgr.24421) 404 : cluster [DBG] pgmap v295: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 519 op/s 2024-04-23T00:59:13.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:12.390091+0000 mgr.smithi153.olhmgd (mgr.24421) 404 : cluster [DBG] pgmap v295: 129 pgs: 129 active+clean; 322 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 519 op/s 2024-04-23T00:59:14.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 235067 6.53 MB/sec execute 563 sec latency 52.071 ms 2024-04-23T00:59:15.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:14.390827+0000 mgr.smithi153.olhmgd (mgr.24421) 405 : cluster [DBG] pgmap v296: 129 pgs: 129 active+clean; 339 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.2 MiB/s wr, 828 op/s 2024-04-23T00:59:15.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 235425 6.53 MB/sec execute 564 sec latency 75.333 ms 2024-04-23T00:59:15.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:14.390827+0000 mgr.smithi153.olhmgd (mgr.24421) 405 : cluster [DBG] pgmap v296: 129 pgs: 129 active+clean; 339 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.2 MiB/s wr, 828 op/s 2024-04-23T00:59:16.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 235865 6.54 MB/sec execute 565 sec latency 47.679 ms 2024-04-23T00:59:17.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 236163 6.53 MB/sec execute 566 sec latency 71.883 ms 2024-04-23T00:59:17.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:16.391269+0000 mgr.smithi153.olhmgd (mgr.24421) 406 : cluster [DBG] pgmap v297: 129 pgs: 129 active+clean; 339 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-04-23T00:59:17.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:16.391269+0000 mgr.smithi153.olhmgd (mgr.24421) 406 : cluster [DBG] pgmap v297: 129 pgs: 129 active+clean; 339 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-04-23T00:59:18.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 236575 6.55 MB/sec execute 567 sec latency 61.434 ms 2024-04-23T00:59:19.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:18.392000+0000 mgr.smithi153.olhmgd (mgr.24421) 407 : cluster [DBG] pgmap v298: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 769 op/s 2024-04-23T00:59:19.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:18.392000+0000 mgr.smithi153.olhmgd (mgr.24421) 407 : cluster [DBG] pgmap v298: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 769 op/s 2024-04-23T00:59:19.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 236911 6.55 MB/sec execute 568 sec latency 36.460 ms 2024-04-23T00:59:20.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 237360 6.57 MB/sec execute 569 sec latency 34.938 ms 2024-04-23T00:59:21.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 237803 6.59 MB/sec execute 570 sec latency 35.845 ms 2024-04-23T00:59:21.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:20.392410+0000 mgr.smithi153.olhmgd (mgr.24421) 408 : cluster [DBG] pgmap v299: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 546 op/s 2024-04-23T00:59:21.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:20.392410+0000 mgr.smithi153.olhmgd (mgr.24421) 408 : cluster [DBG] pgmap v299: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 546 op/s 2024-04-23T00:59:22.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 238157 6.59 MB/sec execute 571 sec latency 53.119 ms 2024-04-23T00:59:23.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 238404 6.59 MB/sec execute 572 sec latency 41.888 ms 2024-04-23T00:59:23.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:22.393870+0000 mgr.smithi153.olhmgd (mgr.24421) 409 : cluster [DBG] pgmap v300: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 546 op/s 2024-04-23T00:59:23.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:22.393870+0000 mgr.smithi153.olhmgd (mgr.24421) 409 : cluster [DBG] pgmap v300: 129 pgs: 129 active+clean; 326 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 546 op/s 2024-04-23T00:59:24.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 238607 6.58 MB/sec execute 573 sec latency 62.961 ms 2024-04-23T00:59:25.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 238898 6.58 MB/sec execute 574 sec latency 71.283 ms 2024-04-23T00:59:25.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:24.394637+0000 mgr.smithi153.olhmgd (mgr.24421) 410 : cluster [DBG] pgmap v301: 129 pgs: 129 active+clean; 333 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-04-23T00:59:25.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:24.394637+0000 mgr.smithi153.olhmgd (mgr.24421) 410 : cluster [DBG] pgmap v301: 129 pgs: 129 active+clean; 333 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-04-23T00:59:26.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 239386 6.59 MB/sec execute 575 sec latency 65.507 ms 2024-04-23T00:59:27.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:26.395525+0000 mgr.smithi153.olhmgd (mgr.24421) 411 : cluster [DBG] pgmap v302: 129 pgs: 129 active+clean; 333 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 464 op/s 2024-04-23T00:59:27.465 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:27.406579+0000 mon.smithi153 (mon.1) 168 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:27.705 INFO:teuthology.orchestra.run.smithi121.stdout: 5 239855 6.59 MB/sec execute 576 sec latency 50.547 ms 2024-04-23T00:59:27.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:26.395525+0000 mgr.smithi153.olhmgd (mgr.24421) 411 : cluster [DBG] pgmap v302: 129 pgs: 129 active+clean; 333 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 464 op/s 2024-04-23T00:59:27.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:27.406579+0000 mon.smithi153 (mon.1) 168 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:28.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 240358 6.61 MB/sec execute 577 sec latency 39.079 ms 2024-04-23T00:59:29.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:28.396156+0000 mgr.smithi153.olhmgd (mgr.24421) 412 : cluster [DBG] pgmap v303: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 700 op/s 2024-04-23T00:59:29.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:28.396156+0000 mgr.smithi153.olhmgd (mgr.24421) 412 : cluster [DBG] pgmap v303: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 700 op/s 2024-04-23T00:59:29.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 240878 6.63 MB/sec execute 578 sec latency 42.609 ms 2024-04-23T00:59:30.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 241301 6.65 MB/sec execute 579 sec latency 32.808 ms 2024-04-23T00:59:31.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 241685 6.65 MB/sec execute 580 sec latency 55.803 ms 2024-04-23T00:59:31.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:30.396468+0000 mgr.smithi153.olhmgd (mgr.24421) 413 : cluster [DBG] pgmap v304: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s 2024-04-23T00:59:31.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:30.396468+0000 mgr.smithi153.olhmgd (mgr.24421) 413 : cluster [DBG] pgmap v304: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s 2024-04-23T00:59:32.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 241952 6.65 MB/sec execute 581 sec latency 51.467 ms 2024-04-23T00:59:33.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 242274 6.65 MB/sec execute 582 sec latency 43.705 ms 2024-04-23T00:59:33.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:32.396852+0000 mgr.smithi153.olhmgd (mgr.24421) 414 : cluster [DBG] pgmap v305: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s 2024-04-23T00:59:33.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:32.396852+0000 mgr.smithi153.olhmgd (mgr.24421) 414 : cluster [DBG] pgmap v305: 129 pgs: 129 active+clean; 325 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s 2024-04-23T00:59:34.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 242726 6.65 MB/sec execute 583 sec latency 38.078 ms 2024-04-23T00:59:35.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 243222 6.66 MB/sec execute 584 sec latency 43.079 ms 2024-04-23T00:59:35.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:34.397429+0000 mgr.smithi153.olhmgd (mgr.24421) 415 : cluster [DBG] pgmap v306: 129 pgs: 129 active+clean; 328 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 714 op/s 2024-04-23T00:59:35.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:34.397429+0000 mgr.smithi153.olhmgd (mgr.24421) 415 : cluster [DBG] pgmap v306: 129 pgs: 129 active+clean; 328 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 714 op/s 2024-04-23T00:59:36.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 243749 6.67 MB/sec execute 585 sec latency 36.600 ms 2024-04-23T00:59:37.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 244305 6.70 MB/sec execute 586 sec latency 42.051 ms 2024-04-23T00:59:37.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:36.397765+0000 mgr.smithi153.olhmgd (mgr.24421) 416 : cluster [DBG] pgmap v307: 129 pgs: 129 active+clean; 328 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 486 op/s 2024-04-23T00:59:37.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:36.397765+0000 mgr.smithi153.olhmgd (mgr.24421) 416 : cluster [DBG] pgmap v307: 129 pgs: 129 active+clean; 328 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 486 op/s 2024-04-23T00:59:38.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 244758 6.72 MB/sec execute 587 sec latency 56.915 ms 2024-04-23T00:59:39.512 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:38.398356+0000 mgr.smithi153.olhmgd (mgr.24421) 417 : cluster [DBG] pgmap v308: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-04-23T00:59:39.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:38.398356+0000 mgr.smithi153.olhmgd (mgr.24421) 417 : cluster [DBG] pgmap v308: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-04-23T00:59:39.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 245179 6.72 MB/sec execute 588 sec latency 38.286 ms 2024-04-23T00:59:40.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 245477 6.72 MB/sec execute 589 sec latency 54.021 ms 2024-04-23T00:59:41.552 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T00:59:41.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 245783 6.72 MB/sec execute 590 sec latency 41.301 ms 2024-04-23T00:59:41.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:40.398754+0000 mgr.smithi153.olhmgd (mgr.24421) 418 : cluster [DBG] pgmap v309: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 538 op/s 2024-04-23T00:59:41.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:40.398754+0000 mgr.smithi153.olhmgd (mgr.24421) 418 : cluster [DBG] pgmap v309: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 538 op/s 2024-04-23T00:59:42.464 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (17m) 9m ago 17m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (16m) 9m ago 16m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (12m) 9m ago 12m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (12m) 9m ago 12m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (19m) 9m ago 19m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (10m) 9m ago 16m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (19m) 9m ago 19m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (16m) 9m ago 16m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T00:59:42.465 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (9m) 9m ago 9m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (9m) 9m ago 9m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (16m) 9m ago 16m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (16m) 9m ago 16m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (15m) 9m ago 15m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (15m) 9m ago 15m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (15m) 9m ago 15m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (14m) 9m ago 14m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (14m) 9m ago 14m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (14m) 9m ago 14m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (13m) 9m ago 13m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T00:59:42.466 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 starting - - - - 2024-04-23T00:59:42.706 INFO:teuthology.orchestra.run.smithi121.stdout: 5 246145 6.72 MB/sec execute 591 sec latency 83.962 ms 2024-04-23T00:59:42.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:41.527147+0000 mgr.smithi153.olhmgd (mgr.24421) 419 : audit [DBG] from='client.15160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:42.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:42.405201+0000 mon.smithi153 (mon.1) 169 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:42.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:41.527147+0000 mgr.smithi153.olhmgd (mgr.24421) 419 : audit [DBG] from='client.15160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:42.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:42.405201+0000 mon.smithi153 (mon.1) 169 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:43.017 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:59:43.017 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T00:59:43.017 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:59:43.017 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:43.017 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T00:59:43.018 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T00:59:43.019 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:59:43.463 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T00:59:43.464 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T00:59:43.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 246584 6.73 MB/sec execute 592 sec latency 48.250 ms 2024-04-23T00:59:43.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:41.981235+0000 mgr.smithi153.olhmgd (mgr.24421) 420 : audit [DBG] from='client.24847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:43.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:42.399131+0000 mgr.smithi153.olhmgd (mgr.24421) 421 : cluster [DBG] pgmap v310: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 538 op/s 2024-04-23T00:59:43.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:42.447575+0000 mgr.smithi153.olhmgd (mgr.24421) 422 : audit [DBG] from='client.15166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:43.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:43.011980+0000 mon.smithi153 (mon.1) 170 : audit [DBG] from='client.? 172.21.15.121:0/2170407785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:59:43.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:41.981235+0000 mgr.smithi153.olhmgd (mgr.24421) 420 : audit [DBG] from='client.24847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:43.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:42.399131+0000 mgr.smithi153.olhmgd (mgr.24421) 421 : cluster [DBG] pgmap v310: 129 pgs: 129 active+clean; 335 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 538 op/s 2024-04-23T00:59:43.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:42.447575+0000 mgr.smithi153.olhmgd (mgr.24421) 422 : audit [DBG] from='client.15166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:43.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:43.011980+0000 mon.smithi153 (mon.1) 170 : audit [DBG] from='client.? 172.21.15.121:0/2170407785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T00:59:44.021 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T00:59:44.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 247027 6.73 MB/sec execute 593 sec latency 48.083 ms 2024-04-23T00:59:44.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:43.458139+0000 mgr.smithi153.olhmgd (mgr.24421) 423 : audit [DBG] from='client.15172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:44.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:44.014736+0000 mon.smithi153 (mon.1) 171 : audit [DBG] from='client.? 172.21.15.121:0/3834154969' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:59:44.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:43.458139+0000 mgr.smithi153.olhmgd (mgr.24421) 423 : audit [DBG] from='client.15172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T00:59:44.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:44.014736+0000 mon.smithi153 (mon.1) 171 : audit [DBG] from='client.? 172.21.15.121:0/3834154969' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T00:59:45.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 247532 6.74 MB/sec execute 594 sec latency 38.858 ms 2024-04-23T00:59:45.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:44.399769+0000 mgr.smithi153.olhmgd (mgr.24421) 424 : cluster [DBG] pgmap v311: 129 pgs: 129 active+clean; 328 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 783 op/s 2024-04-23T00:59:45.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:44.399769+0000 mgr.smithi153.olhmgd (mgr.24421) 424 : cluster [DBG] pgmap v311: 129 pgs: 129 active+clean; 328 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 783 op/s 2024-04-23T00:59:46.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 248022 6.77 MB/sec execute 595 sec latency 38.444 ms 2024-04-23T00:59:47.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 248482 6.78 MB/sec execute 596 sec latency 28.885 ms 2024-04-23T00:59:47.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:46.400108+0000 mgr.smithi153.olhmgd (mgr.24421) 425 : cluster [DBG] pgmap v312: 129 pgs: 129 active+clean; 328 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 532 op/s 2024-04-23T00:59:47.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:46.400108+0000 mgr.smithi153.olhmgd (mgr.24421) 425 : cluster [DBG] pgmap v312: 129 pgs: 129 active+clean; 328 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 532 op/s 2024-04-23T00:59:48.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 248798 6.78 MB/sec execute 597 sec latency 59.448 ms 2024-04-23T00:59:49.577 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:48.400877+0000 mgr.smithi153.olhmgd (mgr.24421) 426 : cluster [DBG] pgmap v313: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 797 op/s 2024-04-23T00:59:49.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:48.400877+0000 mgr.smithi153.olhmgd (mgr.24421) 426 : cluster [DBG] pgmap v313: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 797 op/s 2024-04-23T00:59:49.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 249131 6.78 MB/sec execute 598 sec latency 29.451 ms 2024-04-23T00:59:50.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 249398 6.78 MB/sec execute 599 sec latency 64.198 ms 2024-04-23T00:59:51.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:50.403271+0000 mgr.smithi153.olhmgd (mgr.24421) 427 : cluster [DBG] pgmap v314: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-04-23T00:59:51.707 INFO:teuthology.orchestra.run.smithi121.stdout: 5 cleanup 600 sec 2024-04-23T00:59:51.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:50.403271+0000 mgr.smithi153.olhmgd (mgr.24421) 427 : cluster [DBG] pgmap v314: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-04-23T00:59:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: 0 cleanup 600 sec 2024-04-23T00:59:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:59:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: Operation Count AvgLat MaxLat 2024-04-23T00:59:51.938 INFO:teuthology.orchestra.run.smithi121.stdout: ---------------------------------------- 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: NTCreateX 130602 7.880 300608.043 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Close 95762 0.131 7.133 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Rename 5532 8.281 90.005 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Unlink 26514 5.429 71.272 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Qpathinfo 118253 3.281 300608.705 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Qfileinfo 20563 0.085 6.026 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Qfsinfo 21771 0.588 469.481 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Sfileinfo 10635 9.974 70.511 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Find 45713 0.330 21.282 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: WriteX 64429 19.249 300665.121 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: ReadX 204517 0.010 9.437 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: LockX 422 0.456 5.272 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: UnlockX 422 0.312 2.852 2024-04-23T00:59:51.939 INFO:teuthology.orchestra.run.smithi121.stdout: Flush 9172 0.002 0.020 2024-04-23T00:59:51.940 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:59:51.940 INFO:teuthology.orchestra.run.smithi121.stdout:Throughput 6.78296 MB/sec 5 clients 5 procs max_latency=300665.129 ms 2024-04-23T00:59:51.941 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-23T00:59:51.987 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-23T00:59:52.086 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-23T00:59:52.113 INFO:teuthology.orchestra.run.smithi121.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-23T00:59:52.113 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:59:52.113 INFO:teuthology.orchestra.run.smithi121.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-23T00:59:52.114 INFO:teuthology.orchestra.run.smithi121.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-23T00:59:52.214 INFO:teuthology.orchestra.run.smithi121.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-23T00:59:52.214 INFO:teuthology.orchestra.run.smithi121.stdout:releasing clients 2024-04-23T00:59:53.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:52.403681+0000 mgr.smithi153.olhmgd (mgr.24421) 428 : cluster [DBG] pgmap v315: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-04-23T00:59:53.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:52.403681+0000 mgr.smithi153.olhmgd (mgr.24421) 428 : cluster [DBG] pgmap v315: 129 pgs: 129 active+clean; 340 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 508 op/s 2024-04-23T00:59:54.214 INFO:teuthology.orchestra.run.smithi121.stdout: 5 392 28.09 MB/sec execute 1 sec latency 77.123 ms 2024-04-23T00:59:55.214 INFO:teuthology.orchestra.run.smithi121.stdout: 5 628 30.90 MB/sec execute 2 sec latency 26.899 ms 2024-04-23T00:59:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:54.404372+0000 mgr.smithi153.olhmgd (mgr.24421) 429 : cluster [DBG] pgmap v316: 129 pgs: 129 active+clean; 274 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 734 op/s 2024-04-23T00:59:55.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:54.404372+0000 mgr.smithi153.olhmgd (mgr.24421) 429 : cluster [DBG] pgmap v316: 129 pgs: 129 active+clean; 274 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 734 op/s 2024-04-23T00:59:56.214 INFO:teuthology.orchestra.run.smithi121.stdout: 5 1035 23.97 MB/sec execute 3 sec latency 32.165 ms 2024-04-23T00:59:57.214 INFO:teuthology.orchestra.run.smithi121.stdout: 5 1640 22.34 MB/sec execute 4 sec latency 131.966 ms 2024-04-23T00:59:57.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:56.404768+0000 mgr.smithi153.olhmgd (mgr.24421) 430 : cluster [DBG] pgmap v317: 129 pgs: 129 active+clean; 274 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 489 op/s 2024-04-23T00:59:57.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T00:59:57.406345+0000 mon.smithi153 (mon.1) 172 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:57.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:56.404768+0000 mgr.smithi153.olhmgd (mgr.24421) 430 : cluster [DBG] pgmap v317: 129 pgs: 129 active+clean; 274 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 489 op/s 2024-04-23T00:59:57.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T00:59:57.406345+0000 mon.smithi153 (mon.1) 172 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T00:59:58.215 INFO:teuthology.orchestra.run.smithi121.stdout: 5 cleanup 5 sec 2024-04-23T00:59:58.413 INFO:teuthology.orchestra.run.smithi121.stdout: 0 cleanup 5 sec 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Operation Count AvgLat MaxLat 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: ---------------------------------------- 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: NTCreateX 1826 3.297 38.625 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Close 1603 3.710 296.431 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Rename 57 8.169 32.152 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Unlink 157 2.593 15.342 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Qpathinfo 1544 1.142 17.321 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Qfileinfo 533 11.070 294.152 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Qfsinfo 187 0.375 3.566 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Sfileinfo 188 9.092 19.958 2024-04-23T00:59:58.414 INFO:teuthology.orchestra.run.smithi121.stdout: Find 506 0.292 3.285 2024-04-23T00:59:58.415 INFO:teuthology.orchestra.run.smithi121.stdout: WriteX 2079 0.020 0.056 2024-04-23T00:59:58.415 INFO:teuthology.orchestra.run.smithi121.stdout: ReadX 1889 0.009 0.347 2024-04-23T00:59:58.415 INFO:teuthology.orchestra.run.smithi121.stdout: Flush 156 16.350 301.080 2024-04-23T00:59:58.415 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T00:59:58.415 INFO:teuthology.orchestra.run.smithi121.stdout:Throughput 22.3356 MB/sec 5 clients 5 procs max_latency=131.966 ms 2024-04-23T00:59:58.416 DEBUG:teuthology.parallel:result is None 2024-04-23T00:59:59.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 00:59:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T00:59:58.405385+0000 mgr.smithi153.olhmgd (mgr.24421) 431 : cluster [DBG] pgmap v318: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 15 MiB/s wr, 758 op/s 2024-04-23T00:59:59.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 00:59:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T00:59:58.405385+0000 mgr.smithi153.olhmgd (mgr.24421) 431 : cluster [DBG] pgmap v318: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 15 MiB/s wr, 758 op/s 2024-04-23T01:00:00.578 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:00.000166+0000 mon.smithi121 (mon.0) 833 : cluster [INF] overall HEALTH_OK 2024-04-23T01:00:00.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:00 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:00.000166+0000 mon.smithi121 (mon.0) 833 : cluster [INF] overall HEALTH_OK 2024-04-23T01:00:01.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:00.405805+0000 mgr.smithi153.olhmgd (mgr.24421) 432 : cluster [DBG] pgmap v319: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 12 MiB/s wr, 494 op/s 2024-04-23T01:00:01.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:00.405805+0000 mgr.smithi153.olhmgd (mgr.24421) 432 : cluster [DBG] pgmap v319: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 12 MiB/s wr, 494 op/s 2024-04-23T01:00:03.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:02.406407+0000 mgr.smithi153.olhmgd (mgr.24421) 433 : cluster [DBG] pgmap v320: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 12 MiB/s wr, 494 op/s 2024-04-23T01:00:03.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:02.406407+0000 mgr.smithi153.olhmgd (mgr.24421) 433 : cluster [DBG] pgmap v320: 129 pgs: 129 active+clean; 327 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 12 MiB/s wr, 494 op/s 2024-04-23T01:00:05.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:04.407116+0000 mgr.smithi153.olhmgd (mgr.24421) 434 : cluster [DBG] pgmap v321: 129 pgs: 129 active+clean; 217 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9.7 KiB/s rd, 12 MiB/s wr, 531 op/s 2024-04-23T01:00:05.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:04.407116+0000 mgr.smithi153.olhmgd (mgr.24421) 434 : cluster [DBG] pgmap v321: 129 pgs: 129 active+clean; 217 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9.7 KiB/s rd, 12 MiB/s wr, 531 op/s 2024-04-23T01:00:07.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:06.407721+0000 mgr.smithi153.olhmgd (mgr.24421) 435 : cluster [DBG] pgmap v322: 129 pgs: 129 active+clean; 217 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.7 MiB/s wr, 306 op/s 2024-04-23T01:00:07.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:06.407721+0000 mgr.smithi153.olhmgd (mgr.24421) 435 : cluster [DBG] pgmap v322: 129 pgs: 129 active+clean; 217 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.7 MiB/s wr, 306 op/s 2024-04-23T01:00:09.578 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:08.408401+0000 mgr.smithi153.olhmgd (mgr.24421) 436 : cluster [DBG] pgmap v323: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s rd, 6.7 MiB/s wr, 307 op/s 2024-04-23T01:00:09.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:08.408401+0000 mgr.smithi153.olhmgd (mgr.24421) 436 : cluster [DBG] pgmap v323: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s rd, 6.7 MiB/s wr, 307 op/s 2024-04-23T01:00:11.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:10.408947+0000 mgr.smithi153.olhmgd (mgr.24421) 437 : cluster [DBG] pgmap v324: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:11.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:10.408947+0000 mgr.smithi153.olhmgd (mgr.24421) 437 : cluster [DBG] pgmap v324: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:12.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:12.406453+0000 mon.smithi153 (mon.1) 173 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:12.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:12.406453+0000 mon.smithi153 (mon.1) 173 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:13.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:12.410645+0000 mgr.smithi153.olhmgd (mgr.24421) 438 : cluster [DBG] pgmap v325: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:13.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:12.410645+0000 mgr.smithi153.olhmgd (mgr.24421) 438 : cluster [DBG] pgmap v325: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:14.363 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (18m) 9m ago 18m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (17m) 9m ago 17m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (12m) 9m ago 12m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (12m) 9m ago 12m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (19m) 9m ago 19m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (10m) 9m ago 17m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T01:00:14.986 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (19m) 9m ago 19m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (17m) 9m ago 17m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (10m) 9m ago 10m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (10m) 9m ago 10m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (17m) 9m ago 17m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (16m) 9m ago 16m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (16m) 9m ago 16m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (15m) 9m ago 15m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (15m) 9m ago 15m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (15m) 9m ago 15m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (14m) 9m ago 14m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (14m) 9m ago 14m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (14m) 9m ago 14m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:00:14.987 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 starting - - - - 2024-04-23T01:00:15.355 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:00:15.355 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:00:15.355 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:00:15.355 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:15.355 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:00:15.356 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:00:15.670 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:00:15.670 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:00:15.670 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:00:15.670 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:00:15.671 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T01:00:15.671 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T01:00:15.671 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T01:00:15.671 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:00:15.671 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:00:15.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:14.343676+0000 mgr.smithi153.olhmgd (mgr.24421) 439 : audit [DBG] from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:15.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:14.411273+0000 mgr.smithi153.olhmgd (mgr.24421) 440 : cluster [DBG] pgmap v326: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:15.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:14.661329+0000 mgr.smithi153.olhmgd (mgr.24421) 441 : audit [DBG] from='client.15184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:15.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:15.349760+0000 mon.smithi121 (mon.0) 834 : audit [DBG] from='client.? 172.21.15.121:0/923938992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:00:15.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:14.343676+0000 mgr.smithi153.olhmgd (mgr.24421) 439 : audit [DBG] from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:15.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:14.411273+0000 mgr.smithi153.olhmgd (mgr.24421) 440 : cluster [DBG] pgmap v326: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3.4 KiB/s rd, 65 KiB/s wr, 38 op/s 2024-04-23T01:00:15.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:14.661329+0000 mgr.smithi153.olhmgd (mgr.24421) 441 : audit [DBG] from='client.15184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:15.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:15.349760+0000 mon.smithi121 (mon.0) 834 : audit [DBG] from='client.? 172.21.15.121:0/923938992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:00:16.045 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:00:16.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:14.969935+0000 mgr.smithi153.olhmgd (mgr.24421) 442 : audit [DBG] from='client.15188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:16.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:15.663798+0000 mgr.smithi153.olhmgd (mgr.24421) 443 : audit [DBG] from='client.15196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:16.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:16.039645+0000 mon.smithi121 (mon.0) 835 : audit [DBG] from='client.? 172.21.15.121:0/2948655887' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:00:16.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:14.969935+0000 mgr.smithi153.olhmgd (mgr.24421) 442 : audit [DBG] from='client.15188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:16.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:15.663798+0000 mgr.smithi153.olhmgd (mgr.24421) 443 : audit [DBG] from='client.15196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:16.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:16.039645+0000 mon.smithi121 (mon.0) 835 : audit [DBG] from='client.? 172.21.15.121:0/2948655887' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:00:17.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:16.411857+0000 mgr.smithi153.olhmgd (mgr.24421) 444 : cluster [DBG] pgmap v327: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-23T01:00:17.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:16.411857+0000 mgr.smithi153.olhmgd (mgr.24421) 444 : cluster [DBG] pgmap v327: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 22 KiB/s wr, 1 op/s 2024-04-23T01:00:19.578 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:18.412483+0000 mgr.smithi153.olhmgd (mgr.24421) 445 : cluster [DBG] pgmap v328: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-23T01:00:19.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:18.412483+0000 mgr.smithi153.olhmgd (mgr.24421) 445 : cluster [DBG] pgmap v328: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 22 KiB/s wr, 2 op/s 2024-04-23T01:00:21.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:20.413002+0000 mgr.smithi153.olhmgd (mgr.24421) 446 : cluster [DBG] pgmap v329: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:21.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:20.413002+0000 mgr.smithi153.olhmgd (mgr.24421) 446 : cluster [DBG] pgmap v329: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:23.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:22.413400+0000 mgr.smithi153.olhmgd (mgr.24421) 447 : cluster [DBG] pgmap v330: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:23.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:22.413400+0000 mgr.smithi153.olhmgd (mgr.24421) 447 : cluster [DBG] pgmap v330: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:25.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:24.414119+0000 mgr.smithi153.olhmgd (mgr.24421) 448 : cluster [DBG] pgmap v331: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:25.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:24.414119+0000 mgr.smithi153.olhmgd (mgr.24421) 448 : cluster [DBG] pgmap v331: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:27.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:26.414523+0000 mgr.smithi153.olhmgd (mgr.24421) 449 : cluster [DBG] pgmap v332: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:27.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:27.406773+0000 mon.smithi153 (mon.1) 174 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:27.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:26.414523+0000 mgr.smithi153.olhmgd (mgr.24421) 449 : cluster [DBG] pgmap v332: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:27.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:27.406773+0000 mon.smithi153 (mon.1) 174 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:29.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:29 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:28.415175+0000 mgr.smithi153.olhmgd (mgr.24421) 450 : cluster [DBG] pgmap v333: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:29.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:29 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:28.415175+0000 mgr.smithi153.olhmgd (mgr.24421) 450 : cluster [DBG] pgmap v333: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s 2024-04-23T01:00:31.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:31 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:30.415751+0000 mgr.smithi153.olhmgd (mgr.24421) 451 : cluster [DBG] pgmap v334: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:31.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:31 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:30.415751+0000 mgr.smithi153.olhmgd (mgr.24421) 451 : cluster [DBG] pgmap v334: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:33.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:32.416199+0000 mgr.smithi153.olhmgd (mgr.24421) 452 : cluster [DBG] pgmap v335: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:33.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:32.416199+0000 mgr.smithi153.olhmgd (mgr.24421) 452 : cluster [DBG] pgmap v335: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:35.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:34.416953+0000 mgr.smithi153.olhmgd (mgr.24421) 453 : cluster [DBG] pgmap v336: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:35.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:34.416953+0000 mgr.smithi153.olhmgd (mgr.24421) 453 : cluster [DBG] pgmap v336: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:37.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:37 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:36.417516+0000 mgr.smithi153.olhmgd (mgr.24421) 454 : cluster [DBG] pgmap v337: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:37.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:36.417516+0000 mgr.smithi153.olhmgd (mgr.24421) 454 : cluster [DBG] pgmap v337: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:39.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:38.418296+0000 mgr.smithi153.olhmgd (mgr.24421) 455 : cluster [DBG] pgmap v338: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:00:39.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:38.418296+0000 mgr.smithi153.olhmgd (mgr.24421) 455 : cluster [DBG] pgmap v338: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:00:41.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:40.418690+0000 mgr.smithi153.olhmgd (mgr.24421) 456 : cluster [DBG] pgmap v339: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:41.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:41 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:40.418690+0000 mgr.smithi153.olhmgd (mgr.24421) 456 : cluster [DBG] pgmap v339: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:42.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:42.407081+0000 mon.smithi153 (mon.1) 175 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:42.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:42.407081+0000 mon.smithi153 (mon.1) 175 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:43.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:42.421681+0000 mgr.smithi153.olhmgd (mgr.24421) 457 : cluster [DBG] pgmap v340: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:43.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:42.421681+0000 mgr.smithi153.olhmgd (mgr.24421) 457 : cluster [DBG] pgmap v340: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:45.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:44.422427+0000 mgr.smithi153.olhmgd (mgr.24421) 458 : cluster [DBG] pgmap v341: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:45.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:44.422427+0000 mgr.smithi153.olhmgd (mgr.24421) 458 : cluster [DBG] pgmap v341: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:46.381 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:00:47.006 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 starting - - - - 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (18m) 10m ago 18m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (17m) 10m ago 17m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 starting - - - - 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (13m) 10m ago 13m 12.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (13m) 10m ago 13m 38.8M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:9283 running (20m) 10m ago 20m 359M - 16.2.0 24ecd6d5f14c 2d79d230c558 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (11m) 10m ago 17m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (20m) 10m ago 20m 52.2M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (17m) 10m ago 17m 43.6M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (10m) 10m ago 10m 52.0M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (10m) 10m ago 10m 47.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (17m) 10m ago 17m 17.0M - 0.18.1 e5a616e4b9cf 9b38e6d91f4d 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (17m) 10m ago 17m 402M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:00:47.007 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (16m) 10m ago 16m 313M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (16m) 10m ago 16m 263M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (16m) 10m ago 16m 375M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (15m) 10m ago 15m 384M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (15m) 10m ago 15m 420M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (15m) 10m ago 15m 354M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (14m) 10m ago 14m 491M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:00:47.008 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 starting - - - - 2024-04-23T01:00:47.378 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:00:47.379 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:00:47.380 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-04-23T01:00:47.380 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T01:00:47.380 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:00:47.380 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "1/23 daemons upgraded", 2024-04-23T01:00:47.697 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T01:00:47.698 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:00:47.698 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:00:47.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:46.362682+0000 mgr.smithi153.olhmgd (mgr.24421) 459 : audit [DBG] from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:47.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:46.422825+0000 mgr.smithi153.olhmgd (mgr.24421) 460 : cluster [DBG] pgmap v342: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:47.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:46.680887+0000 mgr.smithi153.olhmgd (mgr.24421) 461 : audit [DBG] from='client.15208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:47.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:47.372923+0000 mon.smithi121 (mon.0) 836 : audit [DBG] from='client.? 172.21.15.121:0/1143700283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:00:47.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:46.362682+0000 mgr.smithi153.olhmgd (mgr.24421) 459 : audit [DBG] from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:47.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:46.422825+0000 mgr.smithi153.olhmgd (mgr.24421) 460 : cluster [DBG] pgmap v342: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:47.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:46.680887+0000 mgr.smithi153.olhmgd (mgr.24421) 461 : audit [DBG] from='client.15208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:47.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:47 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:47.372923+0000 mon.smithi121 (mon.0) 836 : audit [DBG] from='client.? 172.21.15.121:0/1143700283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:00:48.073 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:00:48.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:46.990578+0000 mgr.smithi153.olhmgd (mgr.24421) 462 : audit [DBG] from='client.15212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:48.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:47.690848+0000 mgr.smithi153.olhmgd (mgr.24421) 463 : audit [DBG] from='client.15220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:48.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:48.068047+0000 mon.smithi121 (mon.0) 837 : audit [DBG] from='client.? 172.21.15.121:0/202334486' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:00:48.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:46.990578+0000 mgr.smithi153.olhmgd (mgr.24421) 462 : audit [DBG] from='client.15212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:48.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:47.690848+0000 mgr.smithi153.olhmgd (mgr.24421) 463 : audit [DBG] from='client.15220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:00:48.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:48.068047+0000 mon.smithi121 (mon.0) 837 : audit [DBG] from='client.? 172.21.15.121:0/202334486' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:00:49.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:48.423659+0000 mgr.smithi153.olhmgd (mgr.24421) 464 : cluster [DBG] pgmap v343: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:00:49.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:49 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:48.423659+0000 mgr.smithi153.olhmgd (mgr.24421) 464 : cluster [DBG] pgmap v343: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:00:51.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:50.424034+0000 mgr.smithi153.olhmgd (mgr.24421) 465 : cluster [DBG] pgmap v344: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:51.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:50.424034+0000 mgr.smithi153.olhmgd (mgr.24421) 465 : cluster [DBG] pgmap v344: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:53.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:52.424447+0000 mgr.smithi153.olhmgd (mgr.24421) 466 : cluster [DBG] pgmap v345: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:53.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:52.424447+0000 mgr.smithi153.olhmgd (mgr.24421) 466 : cluster [DBG] pgmap v345: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:55.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:54.425178+0000 mgr.smithi153.olhmgd (mgr.24421) 467 : cluster [DBG] pgmap v346: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:55.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:55 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:54.425178+0000 mgr.smithi153.olhmgd (mgr.24421) 467 : cluster [DBG] pgmap v346: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:57.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:56.425670+0000 mgr.smithi153.olhmgd (mgr.24421) 468 : cluster [DBG] pgmap v347: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:57.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:00:57.407546+0000 mon.smithi153 (mon.1) 176 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:57.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:56.425670+0000 mgr.smithi153.olhmgd (mgr.24421) 468 : cluster [DBG] pgmap v347: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:00:57.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:57 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:00:57.407546+0000 mon.smithi153 (mon.1) 176 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:00:59.632 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:00:59 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:00:58.426313+0000 mgr.smithi153.olhmgd (mgr.24421) 469 : cluster [DBG] pgmap v348: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:00:59.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:00:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:00:58.426313+0000 mgr.smithi153.olhmgd (mgr.24421) 469 : cluster [DBG] pgmap v348: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:01.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:00.426753+0000 mgr.smithi153.olhmgd (mgr.24421) 470 : cluster [DBG] pgmap v349: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:01.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:01 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:00.426753+0000 mgr.smithi153.olhmgd (mgr.24421) 470 : cluster [DBG] pgmap v349: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:03.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:02.427314+0000 mgr.smithi153.olhmgd (mgr.24421) 471 : cluster [DBG] pgmap v350: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:03.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:02.427314+0000 mgr.smithi153.olhmgd (mgr.24421) 471 : cluster [DBG] pgmap v350: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:05.804 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:04.428079+0000 mgr.smithi153.olhmgd (mgr.24421) 472 : cluster [DBG] pgmap v351: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:05.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:05 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:04.428079+0000 mgr.smithi153.olhmgd (mgr.24421) 472 : cluster [DBG] pgmap v351: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:07.121 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.828228+0000 mon.smithi121 (mon.0) 838 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.833905+0000 mon.smithi121 (mon.0) 839 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.879876+0000 mon.smithi153 (mon.1) 177 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.880216+0000 mgr.smithi153.olhmgd (mgr.24421) 473 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.881032+0000 mon.smithi153 (mon.1) 178 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.881260+0000 mgr.smithi153.olhmgd (mgr.24421) 474 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.885112+0000 mon.smithi121 (mon.0) 840 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.894400+0000 mon.smithi153 (mon.1) 179 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.894796+0000 mgr.smithi153.olhmgd (mgr.24421) 475 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:01:07.122 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.896541+0000 mon.smithi153 (mon.1) 180 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.896823+0000 mgr.smithi153.olhmgd (mgr.24421) 476 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.901262+0000 mon.smithi121 (mon.0) 841 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.909544+0000 mon.smithi153 (mon.1) 181 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.911578+0000 mon.smithi153 (mon.1) 182 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.916099+0000 mon.smithi121 (mon.0) 842 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.929146+0000 mon.smithi153 (mon.1) 183 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:07.123 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:06 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.933651+0000 mon.smithi121 (mon.0) 843 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.828228+0000 mon.smithi121 (mon.0) 838 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.833905+0000 mon.smithi121 (mon.0) 839 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.879876+0000 mon.smithi153 (mon.1) 177 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.880216+0000 mgr.smithi153.olhmgd (mgr.24421) 473 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.881032+0000 mon.smithi153 (mon.1) 178 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.881260+0000 mgr.smithi153.olhmgd (mgr.24421) 474 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi121.front.sepia.ceph.com:9093"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.885112+0000 mon.smithi121 (mon.0) 840 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.894400+0000 mon.smithi153 (mon.1) 179 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.894796+0000 mgr.smithi153.olhmgd (mgr.24421) 475 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:01:07.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.896541+0000 mon.smithi153 (mon.1) 180 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.896823+0000 mgr.smithi153.olhmgd (mgr.24421) 476 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi121.front.sepia.ceph.com:3000"}]: dispatch 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.901262+0000 mon.smithi121 (mon.0) 841 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.909544+0000 mon.smithi153 (mon.1) 181 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.911578+0000 mon.smithi153 (mon.1) 182 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.916099+0000 mon.smithi121 (mon.0) 842 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.929146+0000 mon.smithi153 (mon.1) 183 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:07.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.933651+0000 mon.smithi121 (mon.0) 843 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.882 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.909921+0000 mgr.smithi153.olhmgd (mgr.24421) 477 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:05.911872+0000 mgr.smithi153.olhmgd (mgr.24421) 478 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:05.934959+0000 mgr.smithi153.olhmgd (mgr.24421) 479 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:05.935344+0000 mgr.smithi153.olhmgd (mgr.24421) 480 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:06.428386+0000 mgr.smithi153.olhmgd (mgr.24421) 481 : cluster [DBG] pgmap v352: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:07.407501+0000 mon.smithi121 (mon.0) 844 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:07.409314+0000 mon.smithi153 (mon.1) 184 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:07.409895+0000 mon.smithi121 (mon.0) 845 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:07.410787+0000 mon.smithi153 (mon.1) 185 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:01:07.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:07 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:07.411661+0000 mon.smithi153 (mon.1) 186 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:08.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.909921+0000 mgr.smithi153.olhmgd (mgr.24421) 477 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:08.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:05.911872+0000 mgr.smithi153.olhmgd (mgr.24421) 478 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi121.front.sepia.ceph.com:9095"}]: dispatch 2024-04-23T01:01:08.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:05.934959+0000 mgr.smithi153.olhmgd (mgr.24421) 479 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:05.935344+0000 mgr.smithi153.olhmgd (mgr.24421) 480 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:06.428386+0000 mgr.smithi153.olhmgd (mgr.24421) 481 : cluster [DBG] pgmap v352: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:07.407501+0000 mon.smithi121 (mon.0) 844 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:07.409314+0000 mon.smithi153 (mon.1) 184 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:07.409895+0000 mon.smithi121 (mon.0) 845 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:07.410787+0000 mon.smithi153 (mon.1) 185 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:01:08.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:07.411661+0000 mon.smithi153 (mon.1) 186 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:09.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:07.402462+0000 mgr.smithi153.olhmgd (mgr.24421) 482 : cephadm [INF] Upgrade: Updating mgr.smithi121.otezso 2024-04-23T01:01:09.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:07.412505+0000 mgr.smithi153.olhmgd (mgr.24421) 483 : cephadm [INF] Deploying daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:01:09.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:07.402462+0000 mgr.smithi153.olhmgd (mgr.24421) 482 : cephadm [INF] Upgrade: Updating mgr.smithi121.otezso 2024-04-23T01:01:09.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:07.412505+0000 mgr.smithi153.olhmgd (mgr.24421) 483 : cephadm [INF] Deploying daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:01:10.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:08.428961+0000 mgr.smithi153.olhmgd (mgr.24421) 484 : cluster [DBG] pgmap v353: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:10.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:08.428961+0000 mgr.smithi153.olhmgd (mgr.24421) 484 : cluster [DBG] pgmap v353: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:10.429424+0000 mgr.smithi153.olhmgd (mgr.24421) 485 : cluster [DBG] pgmap v354: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.013596+0000 mon.smithi121 (mon.0) 846 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.021675+0000 mon.smithi121 (mon.0) 847 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.034401+0000 mon.smithi153 (mon.1) 187 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.304399+0000 mon.smithi121 (mon.0) 848 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.309876+0000 mon.smithi121 (mon.0) 849 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.354916+0000 mon.smithi121 (mon.0) 850 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.110 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:11.360123+0000 mon.smithi121 (mon.0) 851 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:10.429424+0000 mgr.smithi153.olhmgd (mgr.24421) 485 : cluster [DBG] pgmap v354: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.013596+0000 mon.smithi121 (mon.0) 846 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.021675+0000 mon.smithi121 (mon.0) 847 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.034401+0000 mon.smithi153 (mon.1) 187 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.304399+0000 mon.smithi121 (mon.0) 848 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.309876+0000 mon.smithi121 (mon.0) 849 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.251 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.354916+0000 mon.smithi121 (mon.0) 850 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:12.252 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:11.360123+0000 mon.smithi121 (mon.0) 851 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:13.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:12.407704+0000 mon.smithi153 (mon.1) 188 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:13.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:12 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:12.407704+0000 mon.smithi153 (mon.1) 188 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:13.852 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:13 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:12.429766+0000 mgr.smithi153.olhmgd (mgr.24421) 486 : cluster [DBG] pgmap v355: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:14.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:13 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:12.429766+0000 mgr.smithi153.olhmgd (mgr.24421) 486 : cluster [DBG] pgmap v355: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:16.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:14.430370+0000 mgr.smithi153.olhmgd (mgr.24421) 487 : cluster [DBG] pgmap v356: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:16.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:15.637856+0000 mon.smithi121 (mon.0) 852 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:16.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:15 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:15.643348+0000 mon.smithi121 (mon.0) 853 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:16.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:14.430370+0000 mgr.smithi153.olhmgd (mgr.24421) 487 : cluster [DBG] pgmap v356: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:16.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:15.637856+0000 mon.smithi121 (mon.0) 852 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:16.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:15.643348+0000 mon.smithi121 (mon.0) 853 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:18.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:16.430714+0000 mgr.smithi153.olhmgd (mgr.24421) 488 : cluster [DBG] pgmap v357: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:18.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:17.792401+0000 mon.smithi121 (mon.0) 854 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:18.058 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:17 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:17.798683+0000 mon.smithi121 (mon.0) 855 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:18.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:16.430714+0000 mgr.smithi153.olhmgd (mgr.24421) 488 : cluster [DBG] pgmap v357: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:18.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:17.792401+0000 mon.smithi121 (mon.0) 854 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:18.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:17 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:17.798683+0000 mon.smithi121 (mon.0) 855 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:18.438 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (8m) 1s ago 19m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (19m) 1s ago 19m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (18m) 3s ago 18m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (6m) 1s ago 19m 34.5M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:01:19.105 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (13m) 1s ago 13m 14.7M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (13m) 3s ago 13m 42.0M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (8s) 1s ago 20m 72.4M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283 running (11m) 3s ago 18m 507M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 881334b244af 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (20m) 1s ago 20m 74.8M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (18m) 3s ago 18m 53.7M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (11m) 1s ago 11m 159M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (11m) 3s ago 11m 83.4M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (4m) 1s ago 18m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (13s) 3s ago 18m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (17m) 1s ago 17m 596M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (17m) 1s ago 17m 487M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:01:19.106 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (16m) 1s ago 16m 355M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (16m) 1s ago 16m 481M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (16m) 3s ago 16m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (16m) 3s ago 15m 594M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (15m) 3s ago 15m 510M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (15m) 3s ago 15m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:01:19.107 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 error 1s ago 18m - - 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:18.182338+0000 mon.smithi121 (mon.0) 856 : cluster [DBG] Standby manager daemon smithi121.otezso restarted 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:18.182427+0000 mon.smithi121 (mon.0) 857 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.184457+0000 mon.smithi121 (mon.0) 858 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/crt"}]: dispatch 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.184976+0000 mon.smithi121 (mon.0) 859 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.187233+0000 mon.smithi121 (mon.0) 860 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/key"}]: dispatch 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.188312+0000 mon.smithi121 (mon.0) 861 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.487878+0000 mon.smithi121 (mon.0) 862 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:19.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:18 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.497307+0000 mon.smithi121 (mon.0) 863 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:19.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:18.182338+0000 mon.smithi121 (mon.0) 856 : cluster [DBG] Standby manager daemon smithi121.otezso restarted 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:18.182427+0000 mon.smithi121 (mon.0) 857 : cluster [DBG] Standby manager daemon smithi121.otezso started 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.184457+0000 mon.smithi121 (mon.0) 858 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/crt"}]: dispatch 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.184976+0000 mon.smithi121 (mon.0) 859 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.187233+0000 mon.smithi121 (mon.0) 860 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi121.otezso/key"}]: dispatch 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.188312+0000 mon.smithi121 (mon.0) 861 : audit [DBG] from='mgr.? 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.487878+0000 mon.smithi121 (mon.0) 862 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:19.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:18 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.497307+0000 mon.smithi121 (mon.0) 863 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:19.486 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:01:19.486 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 12, 2024-04-23T01:01:19.487 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:01:19.488 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:01:19.488 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:01:19.811 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr" 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "2/23 daemons upgraded", 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading mgr daemons", 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:01:19.812 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:01:20.068 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.418694+0000 mgr.smithi153.olhmgd (mgr.24421) 489 : audit [DBG] from='client.15232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:20.068 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:18.431587+0000 mgr.smithi153.olhmgd (mgr.24421) 490 : cluster [DBG] pgmap v358: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:20.068 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:18.758554+0000 mgr.smithi153.olhmgd (mgr.24421) 491 : audit [DBG] from='client.15236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:20.068 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:18.867959+0000 mon.smithi121 (mon.0) 864 : cluster [DBG] mgrmap e33: smithi153.olhmgd(active, since 11m), standbys: smithi121.otezso 2024-04-23T01:01:20.068 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:19.480586+0000 mon.smithi121 (mon.0) 865 : audit [DBG] from='client.? 172.21.15.121:0/950460210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:20.204 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:01:20.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.418694+0000 mgr.smithi153.olhmgd (mgr.24421) 489 : audit [DBG] from='client.15232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:20.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:18.431587+0000 mgr.smithi153.olhmgd (mgr.24421) 490 : cluster [DBG] pgmap v358: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:20.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:18.758554+0000 mgr.smithi153.olhmgd (mgr.24421) 491 : audit [DBG] from='client.15236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:20.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:18.867959+0000 mon.smithi121 (mon.0) 864 : cluster [DBG] mgrmap e33: smithi153.olhmgd(active, since 11m), standbys: smithi121.otezso 2024-04-23T01:01:20.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:19 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:19.480586+0000 mon.smithi121 (mon.0) 865 : audit [DBG] from='client.? 172.21.15.121:0/950460210' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:21.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:19.088220+0000 mgr.smithi153.olhmgd (mgr.24421) 492 : audit [DBG] from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:21.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:19.805086+0000 mgr.smithi153.olhmgd (mgr.24421) 493 : audit [DBG] from='client.15248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:21.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:20 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:20.197723+0000 mon.smithi121 (mon.0) 866 : audit [DBG] from='client.? 172.21.15.121:0/4259360822' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:01:21.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:19.088220+0000 mgr.smithi153.olhmgd (mgr.24421) 492 : audit [DBG] from='client.15240 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:21.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:19.805086+0000 mgr.smithi153.olhmgd (mgr.24421) 493 : audit [DBG] from='client.15248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:21.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:20.197723+0000 mon.smithi121 (mon.0) 866 : audit [DBG] from='client.? 172.21.15.121:0/4259360822' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:01:22.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:21 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:20.432087+0000 mgr.smithi153.olhmgd (mgr.24421) 494 : cluster [DBG] pgmap v359: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:22.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:21 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:20.432087+0000 mgr.smithi153.olhmgd (mgr.24421) 494 : cluster [DBG] pgmap v359: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.927525+0000 mon.smithi121 (mon.0) 867 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.935650+0000 mon.smithi121 (mon.0) 868 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.938007+0000 mon.smithi153 (mon.1) 189 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.938909+0000 mon.smithi153 (mon.1) 190 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:21.939685+0000 mgr.smithi153.olhmgd (mgr.24421) 495 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:21.940363+0000 mgr.smithi153.olhmgd (mgr.24421) 496 : cluster [DBG] pgmap v360: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.943506+0000 mon.smithi121 (mon.0) 869 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:21.951425+0000 mon.smithi121 (mon.0) 870 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.187 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.088878+0000 mon.smithi153 (mon.1) 191 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:22.091220+0000 mgr.smithi153.olhmgd (mgr.24421) 497 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:22.091798+0000 mgr.smithi153.olhmgd (mgr.24421) 498 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:22.094901+0000 mgr.smithi153.olhmgd (mgr.24421) 499 : cephadm [INF] Failing over to other MGR 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.095139+0000 mon.smithi153 (mon.1) 192 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.095399+0000 mon.smithi121 (mon.0) 871 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:23.188 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:22 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:22.098949+0000 mon.smithi121 (mon.0) 872 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.927525+0000 mon.smithi121 (mon.0) 867 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.935650+0000 mon.smithi121 (mon.0) 868 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.938007+0000 mon.smithi153 (mon.1) 189 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.938909+0000 mon.smithi153 (mon.1) 190 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:21.939685+0000 mgr.smithi153.olhmgd (mgr.24421) 495 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:21.940363+0000 mgr.smithi153.olhmgd (mgr.24421) 496 : cluster [DBG] pgmap v360: 129 pgs: 129 active+clean; 218 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.943506+0000 mon.smithi121 (mon.0) 869 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:21.951425+0000 mon.smithi121 (mon.0) 870 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' 2024-04-23T01:01:23.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.088878+0000 mon.smithi153 (mon.1) 191 : audit [DBG] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:22.091220+0000 mgr.smithi153.olhmgd (mgr.24421) 497 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:22.091798+0000 mgr.smithi153.olhmgd (mgr.24421) 498 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:22.094901+0000 mgr.smithi153.olhmgd (mgr.24421) 499 : cephadm [INF] Failing over to other MGR 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.095139+0000 mon.smithi153 (mon.1) 192 : audit [INF] from='mgr.24421 172.21.15.153:0/3244479459' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.095399+0000 mon.smithi121 (mon.0) 871 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:23.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:22 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:22.098949+0000 mon.smithi121 (mon.0) 872 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-04-23T01:01:23.995 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:22.934194+0000 mon.smithi121 (mon.0) 873 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.958035+0000 mon.smithi121 (mon.0) 874 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]': finished 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:22.958098+0000 mon.smithi121 (mon.0) 875 : cluster [DBG] mgrmap e34: smithi121.otezso(active, starting, since 0.862038s) 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.959028+0000 mon.smithi121 (mon.0) 876 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.959171+0000 mon.smithi121 (mon.0) 877 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.959677+0000 mon.smithi121 (mon.0) 878 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.959882+0000 mon.smithi121 (mon.0) 879 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.960420+0000 mon.smithi121 (mon.0) 880 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.960654+0000 mon.smithi121 (mon.0) 881 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.960878+0000 mon.smithi121 (mon.0) 882 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.961124+0000 mon.smithi121 (mon.0) 883 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.961403+0000 mon.smithi121 (mon.0) 884 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.961663+0000 mon.smithi121 (mon.0) 885 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.961949+0000 mon.smithi121 (mon.0) 886 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.962234+0000 mon.smithi121 (mon.0) 887 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:01:23.996 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.962531+0000 mon.smithi121 (mon.0) 888 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.962860+0000 mon.smithi121 (mon.0) 889 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.963239+0000 mon.smithi121 (mon.0) 890 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:22.963639+0000 mon.smithi121 (mon.0) 891 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:22.995448+0000 mon.smithi121 (mon.0) 892 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:23.010567+0000 mgr.smithi121.otezso (mgr.15228) 1 : cephadm [INF] inventory: adjusted host smithi153 addr 'smithi153' -> '172.21.15.153' 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.016179+0000 mon.smithi121 (mon.0) 893 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.027018+0000 mon.smithi121 (mon.0) 894 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.055357+0000 mon.smithi121 (mon.0) 895 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.056658+0000 mon.smithi121 (mon.0) 896 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.058345+0000 mon.smithi121 (mon.0) 897 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:23.064527+0000 mon.smithi121 (mon.0) 898 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:23.352242+0000 mgr.smithi121.otezso (mgr.15228) 2 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Bus STARTING 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:23.469470+0000 mgr.smithi121.otezso (mgr.15228) 3 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:23.570709+0000 mgr.smithi121.otezso (mgr.15228) 4 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:01:23.997 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:23 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:23.570905+0000 mgr.smithi121.otezso (mgr.15228) 5 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Bus STARTED 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:22.934194+0000 mon.smithi121 (mon.0) 873 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.958035+0000 mon.smithi121 (mon.0) 874 : audit [INF] from='mgr.24421 ' entity='mgr.smithi153.olhmgd' cmd='[{"prefix": "mgr fail", "who": "smithi153.olhmgd"}]': finished 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:22.958098+0000 mon.smithi121 (mon.0) 875 : cluster [DBG] mgrmap e34: smithi121.otezso(active, starting, since 0.862038s) 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.959028+0000 mon.smithi121 (mon.0) 876 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.959171+0000 mon.smithi121 (mon.0) 877 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.959677+0000 mon.smithi121 (mon.0) 878 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.959882+0000 mon.smithi121 (mon.0) 879 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.960420+0000 mon.smithi121 (mon.0) 880 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.960654+0000 mon.smithi121 (mon.0) 881 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.960878+0000 mon.smithi121 (mon.0) 882 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.961124+0000 mon.smithi121 (mon.0) 883 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:01:24.221 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.961403+0000 mon.smithi121 (mon.0) 884 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.961663+0000 mon.smithi121 (mon.0) 885 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.961949+0000 mon.smithi121 (mon.0) 886 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.962234+0000 mon.smithi121 (mon.0) 887 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.962531+0000 mon.smithi121 (mon.0) 888 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.962860+0000 mon.smithi121 (mon.0) 889 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.963239+0000 mon.smithi121 (mon.0) 890 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:22.963639+0000 mon.smithi121 (mon.0) 891 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:22.995448+0000 mon.smithi121 (mon.0) 892 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:23.010567+0000 mgr.smithi121.otezso (mgr.15228) 1 : cephadm [INF] inventory: adjusted host smithi153 addr 'smithi153' -> '172.21.15.153' 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.016179+0000 mon.smithi121 (mon.0) 893 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.027018+0000 mon.smithi121 (mon.0) 894 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.055357+0000 mon.smithi121 (mon.0) 895 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:24.222 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.056658+0000 mon.smithi121 (mon.0) 896 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.058345+0000 mon.smithi121 (mon.0) 897 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:23.064527+0000 mon.smithi121 (mon.0) 898 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:23.352242+0000 mgr.smithi121.otezso (mgr.15228) 2 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Bus STARTING 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:23.469470+0000 mgr.smithi121.otezso (mgr.15228) 3 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:23.570709+0000 mgr.smithi121.otezso (mgr.15228) 4 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:01:24.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:23.570905+0000 mgr.smithi121.otezso (mgr.15228) 5 : cephadm [INF] [23/Apr/2024:01:01:23] ENGINE Bus STARTED 2024-04-23T01:01:25.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:23.963288+0000 mon.smithi121 (mon.0) 899 : cluster [DBG] mgrmap e35: smithi121.otezso(active, since 1.86722s) 2024-04-23T01:01:25.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:23.978748+0000 mgr.smithi121.otezso (mgr.15228) 6 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:25.194 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:23.963288+0000 mon.smithi121 (mon.0) 899 : cluster [DBG] mgrmap e35: smithi121.otezso(active, since 1.86722s) 2024-04-23T01:01:25.194 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:24 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:23.978748+0000 mgr.smithi121.otezso (mgr.15228) 6 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:26.039 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:24.957862+0000 mon.smithi121 (mon.0) 900 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:01:26.039 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:24.957885+0000 mon.smithi121 (mon.0) 901 : cluster [INF] Cluster is now healthy 2024-04-23T01:01:26.039 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:24.962907+0000 mgr.smithi121.otezso (mgr.15228) 7 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:26.039 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:24.965871+0000 mon.smithi121 (mon.0) 902 : cluster [DBG] mgrmap e36: smithi121.otezso(active, since 2s) 2024-04-23T01:01:26.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:24.957862+0000 mon.smithi121 (mon.0) 900 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:01:26.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:24.957885+0000 mon.smithi121 (mon.0) 901 : cluster [INF] Cluster is now healthy 2024-04-23T01:01:26.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:24.962907+0000 mgr.smithi121.otezso (mgr.15228) 7 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:26.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:24.965871+0000 mon.smithi121 (mon.0) 902 : cluster [DBG] mgrmap e36: smithi121.otezso(active, since 2s) 2024-04-23T01:01:28.000 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:26.963401+0000 mgr.smithi121.otezso (mgr.15228) 8 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:28.000 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:26.971681+0000 mon.smithi121 (mon.0) 903 : cluster [DBG] mgrmap e37: smithi121.otezso(active, since 4s) 2024-04-23T01:01:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:26.963401+0000 mgr.smithi121.otezso (mgr.15228) 8 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:28.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:26.971681+0000 mon.smithi121 (mon.0) 903 : cluster [DBG] mgrmap e37: smithi121.otezso(active, since 4s) 2024-04-23T01:01:29.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:27.975051+0000 mon.smithi121 (mon.0) 904 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:27.978768+0000 mon.smithi121 (mon.0) 905 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:28.667202+0000 mon.smithi121 (mon.0) 906 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:28 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:28.670886+0000 mon.smithi121 (mon.0) 907 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:27.975051+0000 mon.smithi121 (mon.0) 904 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:27.978768+0000 mon.smithi121 (mon.0) 905 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:28.667202+0000 mon.smithi121 (mon.0) 906 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:29.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:28 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:28.670886+0000 mon.smithi121 (mon.0) 907 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:28.963750+0000 mgr.smithi121.otezso (mgr.15228) 9 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:30.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:29.546384+0000 mon.smithi121 (mon.0) 908 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:29.550070+0000 mon.smithi121 (mon.0) 909 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.186761+0000 mon.smithi121 (mon.0) 910 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.190382+0000 mon.smithi121 (mon.0) 911 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:30.291216+0000 mon.smithi121 (mon.0) 912 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.293190+0000 mon.smithi153 (mon.1) 193 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.293624+0000 mon.smithi153 (mon.1) 194 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.296183+0000 mon.smithi153 (mon.1) 195 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:01:30.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:30 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:30.296531+0000 mon.smithi153 (mon.1) 196 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:28.963750+0000 mgr.smithi121.otezso (mgr.15228) 9 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:29.546384+0000 mon.smithi121 (mon.0) 908 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:29.550070+0000 mon.smithi121 (mon.0) 909 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.186761+0000 mon.smithi121 (mon.0) 910 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.190382+0000 mon.smithi121 (mon.0) 911 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:30.291216+0000 mon.smithi121 (mon.0) 912 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.293190+0000 mon.smithi153 (mon.1) 193 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:01:30.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.293624+0000 mon.smithi153 (mon.1) 194 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:30.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.296183+0000 mon.smithi153 (mon.1) 195 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:01:30.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:30 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:30.296531+0000 mon.smithi153 (mon.1) 196 : audit [DBG] from='mgr.? 172.21.15.153:0/1269497460' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:32.200 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:30.964344+0000 mgr.smithi121.otezso (mgr.15228) 10 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-23T01:01:32.200 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:31.198156+0000 mon.smithi121 (mon.0) 913 : cluster [DBG] mgrmap e38: smithi121.otezso(active, since 9s), standbys: smithi153.olhmgd 2024-04-23T01:01:32.201 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:31.198277+0000 mon.smithi121 (mon.0) 914 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:32.578 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:30.964344+0000 mgr.smithi121.otezso (mgr.15228) 10 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-23T01:01:32.578 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:31.198156+0000 mon.smithi121 (mon.0) 913 : cluster [DBG] mgrmap e38: smithi121.otezso(active, since 9s), standbys: smithi153.olhmgd 2024-04-23T01:01:32.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:31.198277+0000 mon.smithi121 (mon.0) 914 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:34.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:32.965004+0000 mgr.smithi121.otezso (mgr.15228) 11 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-23T01:01:34.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.515572+0000 mon.smithi121 (mon.0) 915 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.520798+0000 mon.smithi121 (mon.0) 916 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.521867+0000 mon.smithi121 (mon.0) 917 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.522430+0000 mon.smithi121 (mon.0) 918 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:33.522983+0000 mgr.smithi121.otezso (mgr.15228) 12 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:33.523538+0000 mgr.smithi121.otezso (mgr.15228) 13 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.526512+0000 mon.smithi121 (mon.0) 919 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:33.532100+0000 mon.smithi121 (mon.0) 920 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:33.661613+0000 mgr.smithi121.otezso (mgr.15228) 14 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:01:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:34 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:33.884887+0000 mgr.smithi121.otezso (mgr.15228) 15 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:32.965004+0000 mgr.smithi121.otezso (mgr.15228) 11 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.515572+0000 mon.smithi121 (mon.0) 915 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.520798+0000 mon.smithi121 (mon.0) 916 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.521867+0000 mon.smithi121 (mon.0) 917 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.522430+0000 mon.smithi121 (mon.0) 918 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:33.522983+0000 mgr.smithi121.otezso (mgr.15228) 12 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:33.523538+0000 mgr.smithi121.otezso (mgr.15228) 13 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.526512+0000 mon.smithi121 (mon.0) 919 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:33.532100+0000 mon.smithi121 (mon.0) 920 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:34.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:33.661613+0000 mgr.smithi121.otezso (mgr.15228) 14 : cephadm [INF] Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:01:34.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:33.884887+0000 mgr.smithi121.otezso (mgr.15228) 15 : cephadm [INF] Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:01:35.808 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:34.519315+0000 mon.smithi121 (mon.0) 921 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:01:35.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:35 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:34.519315+0000 mon.smithi121 (mon.0) 921 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:01:36.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:36 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:35.524197+0000 mgr.smithi121.otezso (mgr.15228) 16 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:36.864 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:36 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:35.524197+0000 mgr.smithi121.otezso (mgr.15228) 16 : cluster [DBG] pgmap v10: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:38.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:37.524740+0000 mgr.smithi121.otezso (mgr.15228) 17 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:37.568709+0000 mon.smithi121 (mon.0) 922 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:37.572421+0000 mon.smithi121 (mon.0) 923 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:37.613274+0000 mon.smithi121 (mon.0) 924 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:37.613697+0000 mgr.smithi121.otezso (mgr.15228) 18 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:37.619417+0000 mon.smithi121 (mon.0) 925 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:38.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:38 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:38.050004+0000 mon.smithi121 (mon.0) 926 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:37.524740+0000 mgr.smithi121.otezso (mgr.15228) 17 : cluster [DBG] pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:37.568709+0000 mon.smithi121 (mon.0) 922 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:37.572421+0000 mon.smithi121 (mon.0) 923 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:37.613274+0000 mon.smithi121 (mon.0) 924 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:37.613697+0000 mgr.smithi121.otezso (mgr.15228) 18 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:37.619417+0000 mon.smithi121 (mon.0) 925 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:38.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:38.050004+0000 mon.smithi121 (mon.0) 926 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:39.880 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:39.188905+0000 mon.smithi121 (mon.0) 927 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:39.880 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:39.189827+0000 mon.smithi121 (mon.0) 928 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:01:39.880 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:39.190630+0000 mon.smithi121 (mon.0) 929 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:39.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:39.188905+0000 mon.smithi121 (mon.0) 927 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:01:39.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:39.189827+0000 mon.smithi121 (mon.0) 928 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:01:39.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:39.190630+0000 mon.smithi121 (mon.0) 929 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:40.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:39.188458+0000 mgr.smithi121.otezso (mgr.15228) 19 : cephadm [INF] Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T01:01:40.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:39.191512+0000 mgr.smithi121.otezso (mgr.15228) 20 : cephadm [INF] Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:01:40.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:39.525492+0000 mgr.smithi121.otezso (mgr.15228) 21 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:41.046 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:39.188458+0000 mgr.smithi121.otezso (mgr.15228) 19 : cephadm [INF] Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T01:01:41.046 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:39.191512+0000 mgr.smithi121.otezso (mgr.15228) 20 : cephadm [INF] Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:01:41.046 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:39.525492+0000 mgr.smithi121.otezso (mgr.15228) 21 : cluster [DBG] pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:01:42.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:42 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:41.526042+0000 mgr.smithi121.otezso (mgr.15228) 22 : cluster [DBG] pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:01:42.948 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:41.526042+0000 mgr.smithi121.otezso (mgr.15228) 22 : cluster [DBG] pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:01:44.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:42.756997+0000 mon.smithi121 (mon.0) 930 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:44.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:42.763588+0000 mon.smithi121 (mon.0) 931 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:44.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:42.813835+0000 mon.smithi121 (mon.0) 932 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:44.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:42.756997+0000 mon.smithi121 (mon.0) 930 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:44.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:42.763588+0000 mon.smithi121 (mon.0) 931 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:44.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:43 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:42.813835+0000 mon.smithi121 (mon.0) 932 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:45.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:43.526433+0000 mgr.smithi121.otezso (mgr.15228) 23 : cluster [DBG] pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:01:45.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:43.526433+0000 mgr.smithi121.otezso (mgr.15228) 23 : cluster [DBG] pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:01:46.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:45.527068+0000 mgr.smithi121.otezso (mgr.15228) 24 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:47.070 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:45.527068+0000 mgr.smithi121.otezso (mgr.15228) 24 : cluster [DBG] pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:48.402 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:47.142182+0000 mon.smithi121 (mon.0) 933 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.402 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:47.148055+0000 mon.smithi121 (mon.0) 934 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.402 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:47.527445+0000 mgr.smithi121.otezso (mgr.15228) 25 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:48.402 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:47.781555+0000 mon.smithi121 (mon.0) 935 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.402 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:48 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:47.787770+0000 mon.smithi121 (mon.0) 936 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.469 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:47.142182+0000 mon.smithi121 (mon.0) 933 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.469 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:47.148055+0000 mon.smithi121 (mon.0) 934 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.469 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:47.527445+0000 mgr.smithi121.otezso (mgr.15228) 25 : cluster [DBG] pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:01:48.469 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:47.781555+0000 mon.smithi121 (mon.0) 935 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:48.469 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:47.787770+0000 mon.smithi121 (mon.0) 936 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.097382+0000 mon.smithi121 (mon.0) 937 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.101481+0000 mon.smithi121 (mon.0) 938 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:49.527994+0000 mgr.smithi121.otezso (mgr.15228) 26 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:49.897919+0000 mon.smithi121 (mon.0) 939 : cluster [DBG] Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:49.898033+0000 mon.smithi121 (mon.0) 940 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.900562+0000 mon.smithi153 (mon.1) 197 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.901481+0000 mon.smithi153 (mon.1) 198 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.903858+0000 mon.smithi153 (mon.1) 199 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:01:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:49.904181+0000 mon.smithi153 (mon.1) 200 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.097382+0000 mon.smithi121 (mon.0) 937 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.101481+0000 mon.smithi121 (mon.0) 938 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:49.527994+0000 mgr.smithi121.otezso (mgr.15228) 26 : cluster [DBG] pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:49.897919+0000 mon.smithi121 (mon.0) 939 : cluster [DBG] Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:49.898033+0000 mon.smithi121 (mon.0) 940 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.900562+0000 mon.smithi153 (mon.1) 197 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:01:50.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.901481+0000 mon.smithi153 (mon.1) 198 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:01:50.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.903858+0000 mon.smithi153 (mon.1) 199 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:01:50.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:50 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:49.904181+0000 mon.smithi153 (mon.1) 200 : audit [DBG] from='mgr.? 172.21.15.153:0/2284982905' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:01:50.537 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (9m) 2s ago 19m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (19m) 2s ago 19m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (19m) 4s ago 19m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (7m) 2s ago 19m 34.5M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (14m) 2s ago 14m 15.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (14m) 4s ago 14m 42.2M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (40s) 2s ago 21m 489M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (8s) 4s ago 18m 52.6M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (21m) 2s ago 21m 82.7M 2048M 16.2.0 24ecd6d5f14c ee62d2b36a2d 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (18m) 4s ago 18m 60.0M 2048M 16.2.0 24ecd6d5f14c 92021031a751 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (11m) 2s ago 11m 156M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:01:51.163 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (11m) 4s ago 11m 83.7M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (5m) 2s ago 19m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (45s) 4s ago 18m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (18m) 2s ago 18m 595M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (17m) 2s ago 17m 488M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (17m) 2s ago 17m 355M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (17m) 2s ago 17m 481M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (16m) 4s ago 16m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (16m) 4s ago 16m 594M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (16m) 4s ago 16m 510M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (15m) 4s ago 15m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:01:51.164 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 2s ago 19m - - 2024-04-23T01:01:51.239 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:50.110706+0000 mon.smithi121 (mon.0) 941 : cluster [DBG] mgrmap e39: smithi121.otezso(active, since 28s), standbys: smithi153.olhmgd 2024-04-23T01:01:51.240 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:50.517944+0000 mgr.smithi121.otezso (mgr.15228) 27 : audit [DBG] from='client.15288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:51.240 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:50.838807+0000 mgr.smithi121.otezso (mgr.15228) 28 : audit [DBG] from='client.15292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:51.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:50.110706+0000 mon.smithi121 (mon.0) 941 : cluster [DBG] mgrmap e39: smithi121.otezso(active, since 28s), standbys: smithi153.olhmgd 2024-04-23T01:01:51.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:50.517944+0000 mgr.smithi121.otezso (mgr.15228) 27 : audit [DBG] from='client.15288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:51.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:51 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:50.838807+0000 mgr.smithi121.otezso (mgr.15228) 28 : audit [DBG] from='client.15292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:01:51.575 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 12, 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:01:51.576 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:01:51.911 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr" 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "2/23 daemons upgraded", 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading mon daemons", 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:01:51.912 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:01:52.308 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:01:52.308 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:01:52.308 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.147244+0000 mgr.smithi121.otezso (mgr.15228) 29 : audit [DBG] from='client.15296 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.267691+0000 mon.smithi121 (mon.0) 942 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.272579+0000 mon.smithi121 (mon.0) 943 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.273369+0000 mon.smithi121 (mon.0) 944 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.273999+0000 mon.smithi121 (mon.0) 945 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:51.274557+0000 mgr.smithi121.otezso (mgr.15228) 30 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:51.275055+0000 mgr.smithi121.otezso (mgr.15228) 31 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:51.275234+0000 mgr.smithi121.otezso (mgr.15228) 32 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:01:52.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.279788+0000 mon.smithi121 (mon.0) 946 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.286911+0000 mon.smithi121 (mon.0) 947 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.427354+0000 mon.smithi121 (mon.0) 948 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.430093+0000 mon.smithi121 (mon.0) 949 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:51.430718+0000 mgr.smithi121.otezso (mgr.15228) 33 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.436097+0000 mon.smithi121 (mon.0) 950 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.436782+0000 mon.smithi121 (mon.0) 951 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi121.otezso"}]: dispatch 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.439748+0000 mon.smithi121 (mon.0) 952 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi121.otezso"}]': finished 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.440317+0000 mon.smithi121 (mon.0) 953 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.443033+0000 mon.smithi121 (mon.0) 954 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi153.olhmgd"}]': finished 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.444130+0000 mon.smithi121 (mon.0) 955 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-23T01:01:52.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.568654+0000 mon.smithi121 (mon.0) 956 : audit [DBG] from='client.? 172.21.15.121:0/1729640946' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:52.623 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.147244+0000 mgr.smithi121.otezso (mgr.15228) 29 : audit [DBG] from='client.15296 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.267691+0000 mon.smithi121 (mon.0) 942 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.272579+0000 mon.smithi121 (mon.0) 943 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.273369+0000 mon.smithi121 (mon.0) 944 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.273999+0000 mon.smithi121 (mon.0) 945 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:51.274557+0000 mgr.smithi121.otezso (mgr.15228) 30 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:51.275055+0000 mgr.smithi121.otezso (mgr.15228) 31 : cluster [DBG] pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:51.275234+0000 mgr.smithi121.otezso (mgr.15228) 32 : cluster [DBG] pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.279788+0000 mon.smithi121 (mon.0) 946 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.286911+0000 mon.smithi121 (mon.0) 947 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.427354+0000 mon.smithi121 (mon.0) 948 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.430093+0000 mon.smithi121 (mon.0) 949 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:51.430718+0000 mgr.smithi121.otezso (mgr.15228) 33 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-04-23T01:01:52.624 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.436097+0000 mon.smithi121 (mon.0) 950 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.436782+0000 mon.smithi121 (mon.0) 951 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi121.otezso"}]: dispatch 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.439748+0000 mon.smithi121 (mon.0) 952 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi121.otezso"}]': finished 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.440317+0000 mon.smithi121 (mon.0) 953 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi153.olhmgd"}]: dispatch 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.443033+0000 mon.smithi121 (mon.0) 954 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi153.olhmgd"}]': finished 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.444130+0000 mon.smithi121 (mon.0) 955 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-23T01:01:52.625 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:52 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.568654+0000 mon.smithi121 (mon.0) 956 : audit [DBG] from='client.? 172.21.15.121:0/1729640946' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:51.904959+0000 mgr.smithi121.otezso (mgr.15228) 34 : audit [DBG] from='client.15304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:52.302359+0000 mon.smithi121 (mon.0) 957 : audit [DBG] from='client.? 172.21.15.121:0/1997391334' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:52.915015+0000 mon.smithi121 (mon.0) 958 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:52.915641+0000 mon.smithi121 (mon.0) 959 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:52.916164+0000 mon.smithi121 (mon.0) 960 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:52.916659+0000 mon.smithi121 (mon.0) 961 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:53.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:53.050238+0000 mon.smithi121 (mon.0) 962 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:51.904959+0000 mgr.smithi121.otezso (mgr.15228) 34 : audit [DBG] from='client.15304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:52.302359+0000 mon.smithi121 (mon.0) 957 : audit [DBG] from='client.? 172.21.15.121:0/1997391334' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:52.915015+0000 mon.smithi121 (mon.0) 958 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:52.915641+0000 mon.smithi121 (mon.0) 959 : audit [INF] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:52.916164+0000 mon.smithi121 (mon.0) 960 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:52.916659+0000 mon.smithi121 (mon.0) 961 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:01:53.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:53 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: audit 2024-04-23T01:01:53.050238+0000 mon.smithi121 (mon.0) 962 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:01:54.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:52.910632+0000 mgr.smithi121.otezso (mgr.15228) 35 : cephadm [INF] Upgrade: Updating mon.smithi121 2024-04-23T01:01:54.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cephadm 2024-04-23T01:01:52.917317+0000 mgr.smithi121.otezso (mgr.15228) 36 : cephadm [INF] Deploying daemon mon.smithi121 on smithi121 2024-04-23T01:01:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: cluster 2024-04-23T01:01:53.275504+0000 mgr.smithi121.otezso (mgr.15228) 37 : cluster [DBG] pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:01:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 systemd[1]: Stopping Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:01:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:52.910632+0000 mgr.smithi121.otezso (mgr.15228) 35 : cephadm [INF] Upgrade: Updating mon.smithi121 2024-04-23T01:01:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:01:52.917317+0000 mgr.smithi121.otezso (mgr.15228) 36 : cephadm [INF] Deploying daemon mon.smithi121 on smithi121 2024-04-23T01:01:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:53.275504+0000 mgr.smithi121.otezso (mgr.15228) 37 : cluster [DBG] pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:01:55.084 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T01:01:54.657+0000 7ff4af404700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi121 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-23T01:01:55.084 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:54 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121[35393]: debug 2024-04-23T01:01:54.657+0000 7ff4af404700 -1 mon.smithi121@0(leader) e2 *** Got Signal Terminated *** 2024-04-23T01:01:55.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:55 smithi121 bash[88432]: ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi121 2024-04-23T01:01:55.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:55 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service: Succeeded. 2024-04-23T01:01:55.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:55 smithi121 systemd[1]: Stopped Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:01:55.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:55 smithi121 systemd[1]: Starting Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: set uid:gid to 167:167 (ceph:ceph) 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable), process ceph-mon, pid 7 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: pidfile_write: ignore empty --pid-file 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: load: jerasure load: lrc 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: RocksDB version: 7.9.2 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Git sha 0 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Compile date 2024-04-18 11:37:30 2024-04-23T01:01:56.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: DB SUMMARY 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: DB Session ID: 3J1H976TDCHSU4EPA4LR 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: CURRENT file: CURRENT 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: IDENTITY file: IDENTITY 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: MANIFEST file: MANIFEST-000008 size: 1725 Bytes 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi121/store.db dir, Total Num: 1, files: 000035.sst 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi121/store.db: 000033.log size: 3765562 ; 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.error_if_exists: 0 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.create_if_missing: 0 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.paranoid_checks: 1 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.env: 0x5636dafc1820 2024-04-23T01:01:56.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.fs: PosixFileSystem 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.info_log: 0x5636dcb1f3c0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_file_opening_threads: 16 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.statistics: (nil) 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.use_fsync: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_log_file_size: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.keep_log_file_num: 1000 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.recycle_log_file_num: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_fallocate: 1 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_mmap_reads: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_mmap_writes: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.use_direct_reads: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-23T01:01:56.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.create_missing_column_families: 0 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.db_log_dir: 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.wal_dir: 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.advise_random_on_open: 1 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.db_write_buffer_size: 0 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.write_buffer_manager: 0x5636dbd94140 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-23T01:01:56.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.rate_limiter: (nil) 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.wal_recovery_mode: 2 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enable_thread_tracking: 0 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enable_pipelined_write: 0 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.unordered_write: 0 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.row_cache: None 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.wal_filter: None 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-23T01:01:56.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_ingest_behind: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.two_write_queues: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.manual_wal_flush: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.wal_compression: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.atomic_flush: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.log_readahead_size: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.best_efforts_recovery: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.allow_data_in_errors: 0 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.db_host_id: __hostname__ 2024-04-23T01:01:56.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_background_jobs: 2 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_background_compactions: -1 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_subcompactions: 1 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_total_wal_size: 0 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_open_files: -1 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bytes_per_sync: 0 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-23T01:01:56.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_readahead_size: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_background_flushes: -1 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Compression algorithms supported: 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kZSTD supported: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kXpressCompression supported: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kLZ4HCCompression supported: 1 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kZlibCompression supported: 1 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kSnappyCompression supported: 1 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kLZ4Compression supported: 1 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: kBZip2Compression supported: 0 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000008 2024-04-23T01:01:56.641 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.merge_operator: 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_filter: None 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_filter_factory: None 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.sst_partitioner_factory: None 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5636dcb1f4e0) 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: cache_index_and_filter_blocks: 1 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: pin_top_level_index_and_filter: 1 2024-04-23T01:01:56.642 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: index_type: 0 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: data_block_index_type: 0 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: index_shortening: 1 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: checksum: 4 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: no_block_cache: 0 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_cache: 0x5636dbe351f0 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_cache_name: BinnedLRUCache 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_cache_options: 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: capacity : 536870912 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: num_shard_bits : 4 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: strict_capacity_limit : 0 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: high_pri_pool_ratio: 0.000 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_cache_compressed: (nil) 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: persistent_cache: (nil) 2024-04-23T01:01:56.643 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_size: 4096 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_size_deviation: 10 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_restart_interval: 16 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: index_block_restart_interval: 1 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: metadata_block_size: 4096 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: partition_filters: 0 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: use_delta_encoding: 1 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: filter_policy: bloomfilter 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: whole_key_filtering: 1 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: verify_compression: 0 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: read_amp_bytes_per_bit: 0 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: format_version: 5 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: enable_index_compression: 1 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: block_align: 0 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: max_auto_readahead_size: 262144 2024-04-23T01:01:56.644 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: prepopulate_block_cache: 0 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: initial_auto_readahead_size: 8192 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: num_file_reads_for_auto_readahead: 2 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.write_buffer_size: 33554432 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_write_buffer_number: 2 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression: NoCompression 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression: Disabled 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.prefix_extractor: nullptr 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.num_levels: 7 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-23T01:01:56.645 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.level: 32767 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.strategy: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-23T01:01:56.646 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.enabled: false 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.target_file_size_base: 67108864 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-23T01:01:56.647 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.arena_block_size: 1048576 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.disable_auto_compactions: 0 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-23T01:01:56.648 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-23T01:01:56.649 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-23T01:01:56.649 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-23T01:01:56.649 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-23T01:01:56.649 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-23T01:01:56.649 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.table_properties_collectors: 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.inplace_update_support: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.bloom_locality: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.max_successive_merges: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.paranoid_file_checks: 0 2024-04-23T01:01:56.650 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.force_consistency_checks: 1 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.report_bg_io_stats: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.ttl: 2592000 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enable_blob_files: false 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.min_blob_size: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_file_size: 268435456 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-23T01:01:56.651 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.blob_file_starting_level: 0 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi121/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 37, last_sequence is 18667, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e7d8e368-73a0-4e43-92f2-226feb0c162c 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834116285197, "job": 1, "event": "recovery_started", "wal_files": [33]} 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834116294424, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 3617551, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18668, "largest_seqno": 19412, "table_properties": {"data_size": 3612372, "index_size": 3456, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 5461, "raw_average_key_size": 21, "raw_value_size": 3604826, "raw_average_value_size": 13972, "num_data_blocks": 155, "num_entries": 258, "num_filter_entries": 258, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713834116, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e7d8e368-73a0-4e43-92f2-226feb0c162c", "db_session_id": "3J1H976TDCHSU4EPA4LR", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834116294549, "job": 1, "event": "recovery_finished"} 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/version_set.cc:5047] Creating manifest 40 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi121/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5636dbed4000 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: DB pointer 0x5636dbec2000 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-23T01:01:56.652 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: ** DB Stats ** 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: ** Compaction Stats [default] ** 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: L0 1/0 3.45 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: L6 1/0 30.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-23T01:01:56.653 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Sum 2/0 33.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: ** Compaction Stats [default] ** 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 741.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T01:01:56.654 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Cumulative compaction: 0.00 GB write, 242.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Interval compaction: 0.00 GB write, 242.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Block cache BinnedLRUCache@0x5636dbe351f0#7 capacity: 512.00 MB usage: 4.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: starting mon.smithi121 rank 0 at public addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] at bind addrs [v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi121 fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???) e2 preinit fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).mds e11 new map 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).mds e11 print_map 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: e11 2024-04-23T01:01:56.655 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: legacy client fscid: 1 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Filesystem 'foofs' (1) 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: fs_name foofs 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: epoch 11 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: created 2024-04-23T00:47:26.776148+0000 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: modified 2024-04-23T00:55:10.219465+0000 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: tableserver 0 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: root 0 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: session_timeout 60 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: session_autoclose 300 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: max_file_size 1099511627776 2024-04-23T01:01:56.656 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: max_xattr_size 65536 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: required_client_features {} 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: last_failure 0 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: last_failure_osd_epoch 0 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: max_mds 1 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: in 0 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: up {0=24263} 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: failed 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: damaged 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: stopped 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: data_pools [3] 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: metadata_pool 2 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: inline_data disabled 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: balancer 2024-04-23T01:01:56.657 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: bal_rank_mask -1 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: standby_count_wanted 1 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: [mds.foofs.smithi153.uunfav{0:24263} state up:active seq 117 join_fscid=1 addr [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] compat {c=[1],r=[1],i=[7ff]}] 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: Standby daemons: 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout: [mds.foofs.smithi121.vupjhr{-1:14508} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] compat {c=[1],r=[1],i=[7ff]}] 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 ceph-mon[88653]: mon.smithi121@-1(???).paxosservice(auth 1..41) refresh upgraded, format 0 -> 3 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 bash[88615]: a18fb3dc76a1248394cbf591a1141d2622dcb93071d15fcf0eeb2859d98b8338 2024-04-23T01:01:56.658 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:01:56 smithi121 systemd[1]: Started Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:54.667816+0000 mon.smithi153 (mon.1) 201 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:54.668236+0000 mon.smithi153 (mon.1) 202 : audit [DBG] from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:55.276078+0000 mgr.smithi121.otezso (mgr.15228) 38 : cluster [DBG] pgmap v21: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.709759+0000 mon.smithi121 (mon.0) 1 : cluster [INF] mon.smithi121 calling monitor election 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.724003+0000 mon.smithi121 (mon.0) 2 : cluster [INF] mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T01:01:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.729066+0000 mon.smithi121 (mon.0) 3 : cluster [DBG] monmap e2: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.729677+0000 mon.smithi121 (mon.0) 4 : cluster [DBG] fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.729726+0000 mon.smithi121 (mon.0) 5 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.730895+0000 mon.smithi121 (mon.0) 6 : cluster [DBG] mgrmap e39: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.731626+0000 mon.smithi121 (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.731650+0000 mon.smithi121 (mon.0) 8 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.731662+0000 mon.smithi121 (mon.0) 9 : cluster [WRN] daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:57.740688+0000 mon.smithi121 (mon.0) 10 : cluster [DBG] mgrmap e40: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:01:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:01:57.740766+0000 mon.smithi121 (mon.0) 11 : audit [INF] from='mgr.15228 ' entity='' 2024-04-23T01:01:59.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:01:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:01:58.743045+0000 mon.smithi121 (mon.0) 12 : cluster [DBG] mgrmap e41: smithi121.otezso(active, since 36s), standbys: smithi153.olhmgd 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.543721+0000 mon.smithi121 (mon.0) 13 : cluster [DBG] Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.543796+0000 mon.smithi121 (mon.0) 14 : cluster [DBG] Standby manager daemon smithi153.olhmgd started 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.547186+0000 mon.smithi121 (mon.0) 15 : audit [DBG] from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.547845+0000 mon.smithi121 (mon.0) 16 : audit [DBG] from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.550071+0000 mon.smithi121 (mon.0) 17 : audit [DBG] from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:02:04.728 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.550377+0000 mon.smithi121 (mon.0) 18 : audit [DBG] from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:02:04.729 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.590389+0000 mon.smithi121 (mon.0) 19 : cluster [INF] Active manager daemon smithi121.otezso restarted 2024-04-23T01:02:04.729 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.590689+0000 mon.smithi121 (mon.0) 20 : cluster [INF] Activating manager daemon smithi121.otezso 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.597722+0000 mon.smithi121 (mon.0) 21 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.598104+0000 mon.smithi121 (mon.0) 22 : cluster [DBG] mgrmap e42: smithi121.otezso(active, starting, since 0.00750082s), standbys: smithi153.olhmgd 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.599993+0000 mon.smithi121 (mon.0) 23 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.600159+0000 mon.smithi121 (mon.0) 24 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.601189+0000 mon.smithi121 (mon.0) 25 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:02:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.601497+0000 mon.smithi121 (mon.0) 26 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.601905+0000 mon.smithi121 (mon.0) 27 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.602116+0000 mon.smithi121 (mon.0) 28 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.602358+0000 mon.smithi121 (mon.0) 29 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.602608+0000 mon.smithi121 (mon.0) 30 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.602828+0000 mon.smithi121 (mon.0) 31 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.603053+0000 mon.smithi121 (mon.0) 32 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.603368+0000 mon.smithi121 (mon.0) 33 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.603625+0000 mon.smithi121 (mon.0) 34 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.603813+0000 mon.smithi121 (mon.0) 35 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.607764+0000 mon.smithi121 (mon.0) 36 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.608061+0000 mon.smithi121 (mon.0) 37 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.608284+0000 mon.smithi121 (mon.0) 38 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.608752+0000 mon.smithi121 (mon.0) 39 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T01:02:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:04.627011+0000 mon.smithi121 (mon.0) 40 : cluster [INF] Manager daemon smithi121.otezso is now available 2024-04-23T01:02:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.648384+0000 mon.smithi121 (mon.0) 41 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.673614+0000 mon.smithi121 (mon.0) 42 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.677098+0000 mon.smithi121 (mon.0) 43 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T01:02:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:04.684309+0000 mon.smithi121 (mon.0) 44 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T01:02:06.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:05.614295+0000 mon.smithi121 (mon.0) 45 : cluster [DBG] mgrmap e43: smithi121.otezso(active, since 1.02369s), standbys: smithi153.olhmgd 2024-04-23T01:02:06.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:05.625295+0000 mgr.smithi121.otezso (mgr.34104) 1 : cluster [DBG] pgmap v3: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:07.828 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:06.408931+0000 mgr.smithi121.otezso (mgr.34104) 2 : cephadm [INF] [23/Apr/2024:01:02:06] ENGINE Bus STARTING 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:06.510881+0000 mgr.smithi121.otezso (mgr.34104) 3 : cephadm [INF] [23/Apr/2024:01:02:06] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:06.603364+0000 mgr.smithi121.otezso (mgr.34104) 4 : cluster [DBG] pgmap v4: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:06.609914+0000 mon.smithi121 (mon.0) 46 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:06.609986+0000 mon.smithi121 (mon.0) 47 : cluster [INF] Cluster is now healthy 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:06.621074+0000 mon.smithi121 (mon.0) 48 : cluster [DBG] mgrmap e44: smithi121.otezso(active, since 2s), standbys: smithi153.olhmgd 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:06.625367+0000 mgr.smithi121.otezso (mgr.34104) 5 : cephadm [INF] [23/Apr/2024:01:02:06] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:02:07.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:06.625485+0000 mgr.smithi121.otezso (mgr.34104) 6 : cephadm [INF] [23/Apr/2024:01:02:06] ENGINE Bus STARTED 2024-04-23T01:02:10.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:08.603650+0000 mgr.smithi121.otezso (mgr.34104) 7 : cluster [DBG] pgmap v5: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:10.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:08.624665+0000 mon.smithi121 (mon.0) 49 : cluster [DBG] mgrmap e45: smithi121.otezso(active, since 4s), standbys: smithi153.olhmgd 2024-04-23T01:02:10.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:09.060484+0000 mon.smithi121 (mon.0) 50 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:10.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:09.067937+0000 mon.smithi121 (mon.0) 51 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:10.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:09.709434+0000 mon.smithi121 (mon.0) 52 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:10.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:09.713443+0000 mon.smithi121 (mon.0) 53 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:12.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:10.604008+0000 mgr.smithi121.otezso (mgr.34104) 8 : cluster [DBG] pgmap v6: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:12.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:11.152659+0000 mon.smithi121 (mon.0) 54 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:12.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:11.160234+0000 mon.smithi121 (mon.0) 55 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:13.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:11.993769+0000 mon.smithi121 (mon.0) 56 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:13.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:11.998183+0000 mon.smithi121 (mon.0) 57 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:13.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:12 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:12.604736+0000 mgr.smithi121.otezso (mgr.34104) 9 : cluster [DBG] pgmap v7: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-23T01:02:14.299 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.040649+0000 mon.smithi121 (mon.0) 58 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:14.299 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.046060+0000 mon.smithi121 (mon.0) 59 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:14.299 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.047190+0000 mon.smithi121 (mon.0) 60 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:14.299 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.047840+0000 mon.smithi121 (mon.0) 61 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:13.048382+0000 mgr.smithi121.otezso (mgr.34104) 10 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:13.049015+0000 mgr.smithi121.otezso (mgr.34104) 11 : cluster [DBG] pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.054464+0000 mon.smithi121 (mon.0) 62 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.062853+0000 mon.smithi121 (mon.0) 63 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.166241+0000 mon.smithi121 (mon.0) 64 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.168344+0000 mon.smithi121 (mon.0) 65 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:14.300 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:14 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:13.169244+0000 mon.smithi121 (mon.0) 66 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:14.044560+0000 mon.smithi121 (mon.0) 67 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:14.687184+0000 mgr.smithi121.otezso (mgr.34104) 12 : cephadm [INF] Upgrade: Updating mon.smithi153 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:14.692025+0000 mon.smithi121 (mon.0) 68 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:14.692880+0000 mon.smithi121 (mon.0) 69 : audit [INF] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:14.693545+0000 mon.smithi121 (mon.0) 70 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: audit 2024-04-23T01:02:14.694193+0000 mon.smithi121 (mon.0) 71 : audit [DBG] from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:15.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:15 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cephadm 2024-04-23T01:02:14.695059+0000 mgr.smithi121.otezso (mgr.34104) 13 : cephadm [INF] Deploying daemon mon.smithi153 on smithi153 2024-04-23T01:02:16.189 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: cluster 2024-04-23T01:02:15.049536+0000 mgr.smithi121.otezso (mgr.34104) 14 : cluster [DBG] pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:02:16.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:16 smithi153 systemd[1]: Stopping Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:02:17.034 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T01:02:16.575+0000 7f518fc95700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi153 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-23T01:02:17.035 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:16 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153[38985]: debug 2024-04-23T01:02:16.575+0000 7f518fc95700 -1 mon.smithi153@1(peon) e2 *** Got Signal Terminated *** 2024-04-23T01:02:17.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:17 smithi153 bash[65641]: ceph-004558ac-010a-11ef-bc93-c7b262605968-mon.smithi153 2024-04-23T01:02:18.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:17 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi153.service: Succeeded. 2024-04-23T01:02:18.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:17 smithi153 systemd[1]: Stopped Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:02:18.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:17 smithi153 systemd[1]: Starting Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:02:18.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: set uid:gid to 167:167 (ceph:ceph) 2024-04-23T01:02:18.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable), process ceph-mon, pid 7 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: pidfile_write: ignore empty --pid-file 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: load: jerasure load: lrc 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: RocksDB version: 7.9.2 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Git sha 0 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Compile date 2024-04-18 11:37:30 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: DB SUMMARY 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: DB Session ID: 59USRJKESZPID6BP5XEU 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: CURRENT file: CURRENT 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: IDENTITY file: IDENTITY 2024-04-23T01:02:18.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: MANIFEST file: MANIFEST-000005 size: 1848 Bytes 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi153/store.db dir, Total Num: 1, files: 000035.sst 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi153/store.db: 000033.log size: 3108548 ; 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.error_if_exists: 0 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.create_if_missing: 0 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.paranoid_checks: 1 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.env: 0x55ce9f54b820 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.fs: PosixFileSystem 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.info_log: 0x55cea299b3c0 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_file_opening_threads: 16 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.statistics: (nil) 2024-04-23T01:02:18.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.use_fsync: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_log_file_size: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.keep_log_file_num: 1000 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.recycle_log_file_num: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_fallocate: 1 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_mmap_reads: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_mmap_writes: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.use_direct_reads: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.create_missing_column_families: 0 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.db_log_dir: 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.wal_dir: 2024-04-23T01:02:18.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-23T01:02:18.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.advise_random_on_open: 1 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.db_write_buffer_size: 0 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.write_buffer_manager: 0x55cea1c10140 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.rate_limiter: (nil) 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.wal_recovery_mode: 2 2024-04-23T01:02:18.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enable_thread_tracking: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enable_pipelined_write: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.unordered_write: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.row_cache: None 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.wal_filter: None 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_ingest_behind: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.two_write_queues: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.manual_wal_flush: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.wal_compression: 0 2024-04-23T01:02:18.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.atomic_flush: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.log_readahead_size: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.best_efforts_recovery: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.allow_data_in_errors: 0 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.db_host_id: __hostname__ 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_background_jobs: 2 2024-04-23T01:02:18.837 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_background_compactions: -1 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_subcompactions: 1 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_total_wal_size: 0 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_open_files: -1 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bytes_per_sync: 0 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-23T01:02:18.838 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_readahead_size: 0 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_background_flushes: -1 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Compression algorithms supported: 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kZSTD supported: 0 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kXpressCompression supported: 0 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kLZ4HCCompression supported: 1 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kZlibCompression supported: 1 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kSnappyCompression supported: 1 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kLZ4Compression supported: 1 2024-04-23T01:02:18.839 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: kBZip2Compression supported: 0 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi153/store.db/MANIFEST-000005 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.merge_operator: 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_filter: None 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_filter_factory: None 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.sst_partitioner_factory: None 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cea299b4e0) 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: cache_index_and_filter_blocks: 1 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-23T01:02:18.840 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: pin_top_level_index_and_filter: 1 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: index_type: 0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: data_block_index_type: 0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: index_shortening: 1 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: checksum: 4 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: no_block_cache: 0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_cache: 0x55cea1cb11f0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_cache_name: BinnedLRUCache 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_cache_options: 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: capacity : 536870912 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: num_shard_bits : 4 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: strict_capacity_limit : 0 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: high_pri_pool_ratio: 0.000 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_cache_compressed: (nil) 2024-04-23T01:02:18.841 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: persistent_cache: (nil) 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_size: 4096 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_size_deviation: 10 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_restart_interval: 16 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: index_block_restart_interval: 1 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: metadata_block_size: 4096 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: partition_filters: 0 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: use_delta_encoding: 1 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: filter_policy: bloomfilter 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: whole_key_filtering: 1 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: verify_compression: 0 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: read_amp_bytes_per_bit: 0 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: format_version: 5 2024-04-23T01:02:18.842 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: enable_index_compression: 1 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: block_align: 0 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: max_auto_readahead_size: 262144 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: prepopulate_block_cache: 0 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: initial_auto_readahead_size: 8192 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: num_file_reads_for_auto_readahead: 2 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.write_buffer_size: 33554432 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_write_buffer_number: 2 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression: NoCompression 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression: Disabled 2024-04-23T01:02:18.843 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.prefix_extractor: nullptr 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.num_levels: 7 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-23T01:02:18.844 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.level: 32767 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.strategy: 0 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.enabled: false 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.target_file_size_base: 67108864 2024-04-23T01:02:18.845 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-23T01:02:18.846 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.arena_block_size: 1048576 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.disable_auto_compactions: 0 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-23T01:02:18.847 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.table_properties_collectors: 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.inplace_update_support: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.bloom_locality: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.max_successive_merges: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.paranoid_file_checks: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.force_consistency_checks: 1 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.report_bg_io_stats: 0 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.ttl: 2592000 2024-04-23T01:02:18.848 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enable_blob_files: false 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.min_blob_size: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_file_size: 268435456 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.blob_file_starting_level: 0 2024-04-23T01:02:18.849 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi153/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 37, last_sequence is 20846, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8a4de5c5-65d6-485f-8b6e-7800c53ec8f3 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834138354876, "job": 1, "event": "recovery_started", "wal_files": [33]} 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834138361818, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 2031173, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20851, "largest_seqno": 21141, "table_properties": {"data_size": 2028409, "index_size": 1432, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2427, "raw_average_key_size": 22, "raw_value_size": 2025073, "raw_average_value_size": 18578, "num_data_blocks": 64, "num_entries": 109, "num_filter_entries": 109, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1713834138, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8a4de5c5-65d6-485f-8b6e-7800c53ec8f3", "db_session_id": "59USRJKESZPID6BP5XEU", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713834138361921, "job": 1, "event": "recovery_finished"} 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/version_set.cc:5047] Creating manifest 40 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi153/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cea1d50000 2024-04-23T01:02:18.850 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: DB pointer 0x55cea1d3e000 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: ** DB Stats ** 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: ** Compaction Stats [default] ** 2024-04-23T01:02:18.851 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: L0 1/0 1.94 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: L6 1/0 34.25 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Sum 2/0 36.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 645.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: ** Compaction Stats [default] ** 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 645.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.852 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Cumulative compaction: 0.00 GB write, 170.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Interval compaction: 0.00 GB write, 170.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Block cache BinnedLRUCache@0x55cea1cb11f0#7 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: starting mon.smithi153 rank 1 at public addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] at bind addrs [v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi153 fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???) e2 preinit fsid 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:02:18.853 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).mds e11 new map 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).mds e11 print_map 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: e11 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: legacy client fscid: 1 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Filesystem 'foofs' (1) 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: fs_name foofs 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: epoch 11 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: created 2024-04-23T00:47:26.776148+0000 2024-04-23T01:02:18.854 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: modified 2024-04-23T00:55:10.219465+0000 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: tableserver 0 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: root 0 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: session_timeout 60 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: session_autoclose 300 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: max_file_size 1099511627776 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: max_xattr_size 65536 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: required_client_features {} 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: last_failure 0 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: last_failure_osd_epoch 0 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: max_mds 1 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: in 0 2024-04-23T01:02:18.855 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: up {0=24263} 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: failed 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: damaged 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: stopped 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: data_pools [3] 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: metadata_pool 2 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: inline_data disabled 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: balancer 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: bal_rank_mask -1 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: standby_count_wanted 1 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: [mds.foofs.smithi153.uunfav{0:24263} state up:active seq 117 join_fscid=1 addr [v2:172.21.15.153:6832/2306098855,v1:172.21.15.153:6833/2306098855] compat {c=[1],r=[1],i=[7ff]}] 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.856 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: Standby daemons: 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout: [mds.foofs.smithi121.vupjhr{-1:14508} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.121:6834/171541194,v1:172.21.15.121:6835/171541194] compat {c=[1],r=[1],i=[7ff]}] 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:02:18.857 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-23T01:02:18.858 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 ceph-mon[65847]: mon.smithi153@-1(???).paxosservice(auth 1..42) refresh upgraded, format 0 -> 3 2024-04-23T01:02:18.858 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 bash[65810]: 135da2e1b3abcceb63151aadbb3630cff45312943330739f206542fa21912e77 2024-04-23T01:02:18.858 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:18 smithi153 systemd[1]: Started Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:02:20.773 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mkfs 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:02:20.774 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mkfs 004558ac-010a-11ef-bc93-c7b262605968 2024-04-23T01:02:21.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T01:02:21.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Rados config object exists: conf-nfs.foo 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T01:02:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring node-exporter.smithi121 (dependencies changed)... 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon node-exporter.smithi121 on smithi121 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring node-exporter.smithi153 (dependencies changed)... 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon node-exporter.smithi153 on smithi153 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Updating mgr.smithi121.otezso 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Failing over to other MGR 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: inventory: adjusted host smithi153 addr 'smithi153' -> '172.21.15.153' 2024-04-23T01:02:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:01:23] ENGINE Bus STARTING 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:01:23] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:01:23] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:01:23] ENGINE Bus STARTED 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e38: smithi121.otezso(active, since 9s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v10: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T01:02:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Standby manager daemon smithi153.olhmgd started 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e39: smithi121.otezso(active, since 28s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all mgr 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Updating mon.smithi121 2024-04-23T01:02:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon mon.smithi121 on smithi121 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v21: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mon.smithi121 calling monitor election 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: monmap e2: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: osdmap e75: 8 total, 8 up, 8 in 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e39: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:21.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e40: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.15228 ' entity='' 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e41: smithi121.otezso(active, since 36s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Standby manager daemon smithi153.olhmgd started 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Active manager daemon smithi121.otezso restarted 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Activating manager daemon smithi121.otezso 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: osdmap e76: 8 total, 8 up, 8 in 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e42: smithi121.otezso(active, starting, since 0.00750082s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:02:21.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:02:21.086 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Manager daemon smithi121.otezso is now available 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e43: smithi121.otezso(active, since 1.02369s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v3: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.087 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:02:06] ENGINE Bus STARTING 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:02:06] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v4: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Cluster is now healthy 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e44: smithi121.otezso(active, since 2s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:02:06] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [23/Apr/2024:01:02:06] ENGINE Bus STARTED 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v5: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e45: smithi121.otezso(active, since 4s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v6: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.088 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v7: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-23T01:02:21.089 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Upgrade: Updating mon.smithi153 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Deploying daemon mon.smithi153 on smithi153 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mon.smithi153 calling monitor election 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mon.smithi121 calling monitor election 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: monmap e3: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-23T01:02:21.090 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: osdmap e76: 8 total, 8 up, 8 in 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: mgrmap e45: smithi121.otezso(active, since 15s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.091 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Rados config object exists: conf-nfs.foo 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T01:02:21.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring alertmanager.smithi121 (dependencies changed)... 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon alertmanager.smithi121 on smithi121 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring grafana.smithi121 (dependencies changed)... 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon grafana.smithi121 on smithi121 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring node-exporter.smithi121 (dependencies changed)... 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon node-exporter.smithi121 on smithi121 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring node-exporter.smithi153 (dependencies changed)... 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon node-exporter.smithi153 on smithi153 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Updating mgr.smithi121.otezso 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Need to upgrade myself (mgr.smithi153.olhmgd) 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Failing over to other MGR 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: inventory: adjusted host smithi153 addr 'smithi153' -> '172.21.15.153' 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:01:23] ENGINE Bus STARTING 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:01:23] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:01:23] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:01:23] ENGINE Bus STARTED 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e38: smithi121.otezso(active, since 9s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring prometheus.smithi121 (dependencies changed)... 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Reconfiguring daemon prometheus.smithi121 on smithi121 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v10: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Updating mgr.smithi153.olhmgd 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 193 B/s rd, 0 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:21.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Standby manager daemon smithi153.olhmgd started 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e39: smithi121.otezso(active, since 28s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 174 B/s rd, 0 op/s 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all mgr 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Updating mon.smithi121 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon mon.smithi121 on smithi121 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.15228 172.21.15.121:0/529861690' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v21: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 210 B/s rd, 0 op/s 2024-04-23T01:02:21.138 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mon.smithi121 calling monitor election 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: monmap e2: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: osdmap e75: 8 total, 8 up, 8 in 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e39: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e40: smithi121.otezso(active, since 35s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.15228 ' entity='' 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e41: smithi121.otezso(active, since 36s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Standby manager daemon smithi153.olhmgd restarted 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Standby manager daemon smithi153.olhmgd started 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/crt"}]: dispatch 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-23T01:02:21.139 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi153.olhmgd/key"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.? 172.21.15.153:0/2467196866' entity='mgr.smithi153.olhmgd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Active manager daemon smithi121.otezso restarted 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Activating manager daemon smithi121.otezso 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: osdmap e76: 8 total, 8 up, 8 in 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e42: smithi121.otezso(active, starting, since 0.00750082s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi121.otezso", "id": "smithi121.otezso"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr metadata", "who": "smithi153.olhmgd", "id": "smithi153.olhmgd"}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:02:21.140 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Manager daemon smithi121.otezso is now available 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/mirror_snapshot_schedule"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi121.otezso/trash_purge_schedule"}]: dispatch 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e43: smithi121.otezso(active, since 1.02369s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v3: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:02:06] ENGINE Bus STARTING 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:02:06] ENGINE Serving on http://172.21.15.121:8765 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v4: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.141 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Cluster is now healthy 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e44: smithi121.otezso(active, since 2s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:02:06] ENGINE Serving on https://172.21.15.121:7150 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [23/Apr/2024:01:02:06] ENGINE Bus STARTED 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v5: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e45: smithi121.otezso(active, since 4s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.142 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v6: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v7: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:02:21.143 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v8: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Upgrade: Updating mon.smithi153 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Deploying daemon mon.smithi153 on smithi153 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v9: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: pgmap v11: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:21.144 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi121"}]: dispatch 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mon.smithi153 calling monitor election 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mon.smithi121 calling monitor election 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mon.smithi121 is new leader, mons smithi121,smithi153 in quorum (ranks 0,1) 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: monmap e3: 2 mons at {smithi121=[v2:172.21.15.121:3300/0,v1:172.21.15.121:6789/0],smithi153=[v2:172.21.15.153:3300/0,v1:172.21.15.153:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: osdmap e76: 8 total, 8 up, 8 in 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: mgrmap e45: smithi121.otezso(active, since 15s), standbys: smithi153.olhmgd 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mon metadata", "id": "smithi153"}]: dispatch 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:21.145 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:22.656 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:02:23.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:22 smithi153 ceph-mon[65847]: pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:23.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:22 smithi121 ceph-mon[88653]: pgmap v12: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-23T01:02:23.299 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:02:23.299 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (9m) 12s ago 20m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:02:23.299 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (20m) 12s ago 20m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:02:23.299 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (19m) 14s ago 19m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:02:23.299 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (7m) 12s ago 20m 34.5M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (14m) 12s ago 14m 15.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (14m) 14s ago 14m 42.5M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (72s) 12s ago 21m 470M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (41s) 14s ago 19m 446M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (27s) 12s ago 22m 31.8M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 starting - - - 2048M 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (12m) 12s ago 12m 154M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (12m) 14s ago 12m 83.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (5m) 12s ago 20m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (78s) 14s ago 19m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:02:23.300 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (18m) 12s ago 18m 595M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (18m) 12s ago 18m 488M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (18m) 12s ago 18m 355M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (17m) 12s ago 17m 481M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (17m) 14s ago 17m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (17m) 14s ago 17m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (16m) 14s ago 16m 511M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (16m) 14s ago 16m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:02:23.301 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 12s ago 19m - - 2024-04-23T01:02:23.699 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:02:23.700 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:23.701 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:02:23.701 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 10, 2024-04-23T01:02:23.701 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 4 2024-04-23T01:02:23.701 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:02:23.701 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr" 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "3/23 daemons upgraded", 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading mon daemons", 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:02:24.039 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:02:24.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:23 smithi121 ceph-mon[88653]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:24.057 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:23 smithi121 ceph-mon[88653]: from='client.34142 172.21.15.121:0/659074006' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:24.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:23 smithi153 ceph-mon[65847]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:24.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:23 smithi153 ceph-mon[65847]: from='client.34142 172.21.15.121:0/659074006' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:24.429 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:24.429 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:24.429 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 196 B/s rd, 0 op/s 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='client.34150 172.21.15.121:0/22803952' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: pgmap v13: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 196 B/s rd, 0 op/s 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='client.34150 172.21.15.121:0/22803952' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:26.066 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:25 smithi153 ceph-mon[65847]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:26.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:25 smithi121 ceph-mon[88653]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:27.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:26 smithi153 ceph-mon[65847]: pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:27.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:26 smithi121 ceph-mon[88653]: pgmap v14: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:29.041 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: pgmap v15: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:29.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:29 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:29.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:29 smithi121 ceph-mon[88653]: pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-04-23T01:02:29.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:29 smithi121 ceph-mon[88653]: pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:29.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:29 smithi121 ceph-mon[88653]: Reconfiguring mon.smithi121 (monmap changed)... 2024-04-23T01:02:29.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:29 smithi121 ceph-mon[88653]: Reconfiguring daemon mon.smithi121 on smithi121 2024-04-23T01:02:30.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:29 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:02:30.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:29 smithi153 ceph-mon[65847]: pgmap v16: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-04-23T01:02:30.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:29 smithi153 ceph-mon[65847]: pgmap v17: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:30.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:29 smithi153 ceph-mon[65847]: Reconfiguring mon.smithi121 (monmap changed)... 2024-04-23T01:02:30.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:29 smithi153 ceph-mon[65847]: Reconfiguring daemon mon.smithi121 on smithi121 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: Reconfiguring mgr.smithi121.otezso (monmap changed)... 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: Reconfiguring daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:02:31.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:30 smithi153 ceph-mon[65847]: pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:31.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:31.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:31.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: Reconfiguring mgr.smithi121.otezso (monmap changed)... 2024-04-23T01:02:31.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi121.otezso", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:02:31.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:02:31.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:31.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: Reconfiguring daemon mgr.smithi121.otezso on smithi121 2024-04-23T01:02:31.367 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:30 smithi121 ceph-mon[88653]: pgmap v18: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:32.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:32.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:32.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: Reconfiguring crash.smithi121 (monmap changed)... 2024-04-23T01:02:32.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:32.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:32.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:32 smithi121 ceph-mon[88653]: Reconfiguring daemon crash.smithi121 on smithi121 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: Reconfiguring crash.smithi121 (monmap changed)... 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:32.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:32 smithi153 ceph-mon[65847]: Reconfiguring daemon crash.smithi121 on smithi121 2024-04-23T01:02:33.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:33 smithi153 ceph-mon[65847]: pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:33.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:33.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:33.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T01:02:33.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:33.837 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:33 smithi121 ceph-mon[88653]: pgmap v19: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:33.837 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:33.838 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:33.838 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T01:02:33.838 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:34.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:34 smithi121 ceph-mon[88653]: Reconfiguring osd.0 (monmap changed)... 2024-04-23T01:02:34.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:34 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.0 on smithi121 2024-04-23T01:02:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:34 smithi153 ceph-mon[65847]: Reconfiguring osd.0 (monmap changed)... 2024-04-23T01:02:34.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:34 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.0 on smithi121 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: Reconfiguring osd.1 (monmap changed)... 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:35.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:35 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.1 on smithi121 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: pgmap v20: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: Reconfiguring osd.1 (monmap changed)... 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:35.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:35 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.1 on smithi121 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: pgmap v21: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: Reconfiguring osd.2 (monmap changed)... 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:37.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:37 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.2 on smithi121 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: pgmap v21: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 219 B/s rd, 0 op/s 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: Reconfiguring osd.2 (monmap changed)... 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:37.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:37 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.2 on smithi121 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: Reconfiguring osd.3 (monmap changed)... 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.3 on smithi121 2024-04-23T01:02:39.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:39 smithi153 ceph-mon[65847]: pgmap v22: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:02:39.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: Reconfiguring osd.3 (monmap changed)... 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.3 on smithi121 2024-04-23T01:02:39.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:39 smithi121 ceph-mon[88653]: pgmap v22: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:02:41.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: Reconfiguring mds.foofs.smithi121.vupjhr (monmap changed)... 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: Reconfiguring daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T01:02:41.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:40 smithi121 ceph-mon[88653]: pgmap v23: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: Reconfiguring mds.foofs.smithi121.vupjhr (monmap changed)... 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: Reconfiguring daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T01:02:41.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:40 smithi153 ceph-mon[65847]: pgmap v23: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:42.553 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:42.553 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:42.553 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: Reconfiguring crash.smithi153 (monmap changed)... 2024-04-23T01:02:42.553 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:42.554 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:42.554 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:42 smithi153 ceph-mon[65847]: Reconfiguring daemon crash.smithi153 on smithi153 2024-04-23T01:02:42.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:42.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:42.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: Reconfiguring crash.smithi153 (monmap changed)... 2024-04-23T01:02:42.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:42.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:42.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:42 smithi121 ceph-mon[88653]: Reconfiguring daemon crash.smithi153 on smithi153 2024-04-23T01:02:43.732 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: pgmap v24: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:43.732 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:43.732 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:43.732 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:02:43.732 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:02:43.733 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:43.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: pgmap v24: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:43.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:43.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:43.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi153.olhmgd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-23T01:02:43.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-23T01:02:43.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:44.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:44 smithi153 ceph-mon[65847]: Reconfiguring mgr.smithi153.olhmgd (monmap changed)... 2024-04-23T01:02:44.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:44 smithi153 ceph-mon[65847]: Reconfiguring daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:02:44.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:44 smithi121 ceph-mon[88653]: Reconfiguring mgr.smithi153.olhmgd (monmap changed)... 2024-04-23T01:02:44.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:44 smithi121 ceph-mon[88653]: Reconfiguring daemon mgr.smithi153.olhmgd on smithi153 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: pgmap v25: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: Reconfiguring mon.smithi153 (monmap changed)... 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:45.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:45 smithi153 ceph-mon[65847]: Reconfiguring daemon mon.smithi153 on smithi153 2024-04-23T01:02:45.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: pgmap v25: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: Reconfiguring mon.smithi153 (monmap changed)... 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:45 smithi121 ceph-mon[88653]: Reconfiguring daemon mon.smithi153 on smithi153 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: Reconfiguring osd.4 (monmap changed)... 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.4 on smithi153 2024-04-23T01:02:47.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:46 smithi153 ceph-mon[65847]: pgmap v26: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: Reconfiguring osd.4 (monmap changed)... 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.4 on smithi153 2024-04-23T01:02:47.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:46 smithi121 ceph-mon[88653]: pgmap v26: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: Reconfiguring osd.5 (monmap changed)... 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:48 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.5 on smithi153 2024-04-23T01:02:48.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:48.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:48.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: Reconfiguring osd.5 (monmap changed)... 2024-04-23T01:02:48.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T01:02:48.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:48.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:48 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.5 on smithi153 2024-04-23T01:02:49.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:49 smithi153 ceph-mon[65847]: pgmap v27: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:49.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:49.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:49.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T01:02:49.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:49.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:49 smithi121 ceph-mon[88653]: pgmap v27: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:49.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:49.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:49.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T01:02:49.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:50.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:50 smithi153 ceph-mon[65847]: Reconfiguring osd.6 (monmap changed)... 2024-04-23T01:02:50.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:50 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.6 on smithi153 2024-04-23T01:02:50.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:50.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:50 smithi121 ceph-mon[88653]: Reconfiguring osd.6 (monmap changed)... 2024-04-23T01:02:50.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:50 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.6 on smithi153 2024-04-23T01:02:50.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:02:51.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: pgmap v28: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:51.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:51.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:51.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: Reconfiguring osd.7 (monmap changed)... 2024-04-23T01:02:51.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T01:02:51.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:51.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:51 smithi153 ceph-mon[65847]: Reconfiguring daemon osd.7 on smithi153 2024-04-23T01:02:51.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: pgmap v28: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: Reconfiguring osd.7 (monmap changed)... 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:51.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:51 smithi121 ceph-mon[88653]: Reconfiguring daemon osd.7 on smithi153 2024-04-23T01:02:53.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: Reconfiguring mds.foofs.smithi153.uunfav (monmap changed)... 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: Reconfiguring daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T01:02:53.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:53 smithi121 ceph-mon[88653]: pgmap v29: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: Reconfiguring mds.foofs.smithi153.uunfav (monmap changed)... 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: Reconfiguring daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T01:02:53.436 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:53 smithi153 ceph-mon[65847]: pgmap v29: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all mon 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi121"}]: dispatch 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi121"}]': finished 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi153"}]: dispatch 2024-04-23T01:02:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi153"}]': finished 2024-04-23T01:02:54.778 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all mon 2024-04-23T01:02:54.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi121"}]: dispatch 2024-04-23T01:02:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi121"}]': finished 2024-04-23T01:02:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi153"}]: dispatch 2024-04-23T01:02:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi153"}]': finished 2024-04-23T01:02:55.458 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (10m) 44s ago 21m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (21m) 44s ago 21m 6391k - 16.2.0 24ecd6d5f14c b40743a07db6 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (20m) 31s ago 20m 6405k - 16.2.0 24ecd6d5f14c c98f0e71daca 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (8m) 44s ago 20m 34.5M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (15m) 44s ago 15m 15.4M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (15m) 31s ago 15m 42.6M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (105s) 44s ago 22m 470M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (73s) 31s ago 20m 446M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (59s) 44s ago 22m 31.8M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (37s) 31s ago 19m 23.0M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (12m) 44s ago 12m 154M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (12m) 31s ago 12m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:02:55.459 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (6m) 44s ago 20m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (110s) 31s ago 19m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (19m) 44s ago 19m 595M 4096M 16.2.0 24ecd6d5f14c b9b1e94e29f4 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (18m) 44s ago 18m 488M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (18m) 44s ago 18m 355M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (18m) 44s ago 18m 481M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (17m) 31s ago 17m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (17m) 31s ago 17m 594M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (17m) 31s ago 17m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (16m) 31s ago 16m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:02:55.460 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 44s ago 20m - - 2024-04-23T01:02:55.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:55 smithi153 ceph-mon[65847]: pgmap v30: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:55.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:55 smithi153 ceph-mon[65847]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:55.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:55.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:55.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:55.852 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 10, 2024-04-23T01:02:55.853 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 4 2024-04-23T01:02:55.854 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:02:55.854 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:02:55.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:55 smithi121 ceph-mon[88653]: pgmap v30: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:55.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:55 smithi121 ceph-mon[88653]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:55.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:55.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi121", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:02:55.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "4/23 daemons upgraded", 2024-04-23T01:02:56.180 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading crash daemons", 2024-04-23T01:02:56.181 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:02:56.181 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:02:56.567 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:02:56.567 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:02:56.567 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:02:56.825 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:56 smithi121 ceph-mon[88653]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:56.825 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:56 smithi121 ceph-mon[88653]: Upgrade: Updating crash.smithi121 (1/2) 2024-04-23T01:02:56.825 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:56 smithi121 ceph-mon[88653]: Deploying daemon crash.smithi121 on smithi121 2024-04-23T01:02:56.825 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:56 smithi121 ceph-mon[88653]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:56.825 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:56 smithi121 ceph-mon[88653]: from='client.34166 172.21.15.121:0/1671659030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:56.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:56 smithi153 ceph-mon[65847]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:56.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:56 smithi153 ceph-mon[65847]: Upgrade: Updating crash.smithi121 (1/2) 2024-04-23T01:02:56.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:56 smithi153 ceph-mon[65847]: Deploying daemon crash.smithi121 on smithi121 2024-04-23T01:02:56.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:56 smithi153 ceph-mon[65847]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:56.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:56 smithi153 ceph-mon[65847]: from='client.34166 172.21.15.121:0/1671659030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:02:57.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:57 smithi153 ceph-mon[65847]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:57.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:57 smithi153 ceph-mon[65847]: pgmap v31: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:57.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:57 smithi153 ceph-mon[65847]: from='client.34174 172.21.15.121:0/2953907625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:02:57.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:57 smithi121 ceph-mon[88653]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:02:57.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:57 smithi121 ceph-mon[88653]: pgmap v31: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:02:57.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:57 smithi121 ceph-mon[88653]: from='client.34174 172.21.15.121:0/2953907625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:02:59.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:59 smithi121 ceph-mon[88653]: pgmap v32: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:59.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:59.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:02:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:59.761 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:59 smithi153 ceph-mon[65847]: pgmap v32: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:02:59.761 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:02:59.762 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:02:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: Upgrade: Updating crash.smithi153 (2/2) 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: Deploying daemon crash.smithi153 on smithi153 2024-04-23T01:03:01.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:01 smithi153 ceph-mon[65847]: pgmap v33: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:01.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: Upgrade: Updating crash.smithi153 (2/2) 2024-04-23T01:03:01.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:01.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi153", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-23T01:03:01.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:01.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: Deploying daemon crash.smithi153 on smithi153 2024-04-23T01:03:01.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:01 smithi121 ceph-mon[88653]: pgmap v33: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:03.633 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:03 smithi153 ceph-mon[65847]: pgmap v34: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:03.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:03 smithi121 ceph-mon[88653]: pgmap v34: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:04.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:04.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:04.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:04.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:05.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:05.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:05.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:05.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:05.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:05 smithi153 ceph-mon[65847]: pgmap v35: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:05.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:05 smithi121 ceph-mon[88653]: pgmap v35: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:08.017 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:07 smithi121 ceph-mon[88653]: pgmap v36: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:03:08.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:07 smithi153 ceph-mon[65847]: pgmap v36: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:03:09.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:08 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:08 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:08 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:08 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:08 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:08 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:08 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:08 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.875 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:09 smithi153 ceph-mon[65847]: pgmap v37: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:09.875 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:09.875 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:10.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:09 smithi121 ceph-mon[88653]: pgmap v37: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:10.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:10.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:11.401 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:11 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:11.401 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:11 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:11.401 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:11 smithi153 ceph-mon[65847]: pgmap v38: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:11.613 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:11 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:11.613 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:11 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:11.614 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:11 smithi121 ceph-mon[88653]: pgmap v38: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:12.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:12 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:12.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:12 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:13.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:12 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:13.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:12 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:14.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:13 smithi153 ceph-mon[65847]: pgmap v39: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:14.100 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:13 smithi121 ceph-mon[88653]: pgmap v39: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:15.707 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: pgmap v40: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:15.707 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.707 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.707 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: pgmap v41: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: pgmap v42: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all crash 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi121"}]: dispatch 2024-04-23T01:03:15.708 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi121"}]': finished 2024-04-23T01:03:15.709 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi153"}]: dispatch 2024-04-23T01:03:15.709 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi153"}]': finished 2024-04-23T01:03:15.709 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-23T01:03:15.709 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-23T01:03:15.709 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:15 smithi121 ceph-mon[88653]: Upgrade: osd.0 is safe to restart 2024-04-23T01:03:15.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: pgmap v40: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:15.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: pgmap v41: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: pgmap v42: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:15.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all crash 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi121"}]: dispatch 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi121"}]': finished 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi153"}]: dispatch 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi153"}]': finished 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-23T01:03:15.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:15 smithi153 ceph-mon[65847]: Upgrade: osd.0 is safe to restart 2024-04-23T01:03:17.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: Upgrade: Updating osd.0 2024-04-23T01:03:17.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:17.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T01:03:17.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:17.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: Deploying daemon osd.0 on smithi121 2024-04-23T01:03:17.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:17 smithi153 ceph-mon[65847]: pgmap v43: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: Upgrade: Updating osd.0 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: Deploying daemon osd.0 on smithi121 2024-04-23T01:03:17.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:17 smithi121 ceph-mon[88653]: pgmap v43: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:19.477 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:19 smithi121 systemd[1]: Stopping Ceph osd.0 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:03:19.478 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T01:03:19.357+0000 7f7ea6c92700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:03:19.478 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T01:03:19.357+0000 7f7ea6c92700 -1 osd.0 76 *** Got signal Terminated *** 2024-04-23T01:03:19.478 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0[46073]: debug 2024-04-23T01:03:19.357+0000 7f7ea6c92700 -1 osd.0 76 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:03:19.814 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:19 smithi121 ceph-mon[88653]: pgmap v44: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:19.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:19 smithi153 ceph-mon[65847]: pgmap v44: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:20.133 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:19 smithi121 bash[95054]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.0 2024-04-23T01:03:20.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:20.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 failed (root=default,host=smithi121) (connection refused reported by osd.6) 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:20 smithi153 ceph-mon[65847]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 failed (root=default,host=smithi121) (connection refused reported by osd.6) 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.5 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.1 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.2 2024-04-23T01:03:20.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.3 2024-04-23T01:03:20.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.4 2024-04-23T01:03:20.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.6 2024-04-23T01:03:20.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:20 smithi121 ceph-mon[88653]: osd.0 reported immediately failed by osd.7 2024-04-23T01:03:21.782 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:21 smithi121 ceph-mon[88653]: pgmap v45: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:21.782 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:21 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:03:21.783 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:21 smithi121 ceph-mon[88653]: osdmap e77: 8 total, 7 up, 8 in 2024-04-23T01:03:21.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:21 smithi153 ceph-mon[65847]: pgmap v45: 129 pgs: 129 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:03:21.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:21 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:03:21.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:21 smithi153 ceph-mon[65847]: osdmap e77: 8 total, 7 up, 8 in 2024-04-23T01:03:22.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:21 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.0.service: Succeeded. 2024-04-23T01:03:22.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:21 smithi121 systemd[1]: Stopped Ceph osd.0 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:03:22.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:21 smithi121 systemd[1]: Starting Ceph osd.0 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:03:22.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:22 smithi153 ceph-mon[65847]: osdmap e78: 8 total, 7 up, 8 in 2024-04-23T01:03:22.872 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:22 smithi121 ceph-mon[88653]: osdmap e78: 8 total, 7 up, 8 in 2024-04-23T01:03:23.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:23 smithi121 ceph-mon[88653]: pgmap v48: 129 pgs: 49 peering, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:23.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:23 smithi121 ceph-mon[88653]: Health check failed: Reduced data availability: 7 pgs inactive, 49 pgs peering (PG_AVAILABILITY) 2024-04-23T01:03:23.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-23T01:03:23.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-23T01:03:23.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-23T01:03:23.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-23T01:03:23.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-23T01:03:23.635 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-23T01:03:23.635 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:23 smithi121 bash[95399]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-23T01:03:23.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:23 smithi153 ceph-mon[65847]: pgmap v48: 129 pgs: 49 peering, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:23.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:23 smithi153 ceph-mon[65847]: Health check failed: Reduced data availability: 7 pgs inactive, 49 pgs peering (PG_AVAILABILITY) 2024-04-23T01:03:25.133 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:24 smithi121 ceph-osd[95738]: -- 172.21.15.121:0/1698569726 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56464bfc11e0 con 0x56464bf90c00 2024-04-23T01:03:25.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:24 smithi121 bash[95700]: d5d04a0a3e390292fdeea52dd7beea0eca64d042c0b804029e46c66dcd5d3c2c 2024-04-23T01:03:25.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:24 smithi121 systemd[1]: Started Ceph osd.0 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:03:25.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:25 smithi121 ceph-mon[88653]: pgmap v49: 129 pgs: 49 peering, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:25.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:25.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:25.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:25.634 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:03:25.364+0000 7fdbecf317c0 -1 Falling back to public interface 2024-04-23T01:03:25.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:25 smithi153 ceph-mon[65847]: pgmap v49: 129 pgs: 49 peering, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-04-23T01:03:25.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:25.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:25.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:26.768 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:26 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 67/438 objects degraded (15.297%), 18 pgs degraded (PG_DEGRADED) 2024-04-23T01:03:26.768 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:26 smithi121 ceph-mon[88653]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 49 pgs peering) 2024-04-23T01:03:26.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:26 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 67/438 objects degraded (15.297%), 18 pgs degraded (PG_DEGRADED) 2024-04-23T01:03:26.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:26 smithi153 ceph-mon[65847]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 49 pgs peering) 2024-04-23T01:03:26.914 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:03:27.598 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (10m) 17s ago 21m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (29s) 17s ago 21m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (24s) 19s ago 20m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (8m) 17s ago 21m 34.5M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (15m) 17s ago 15m 15.7M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (15m) 19s ago 15m 42.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (2m) 17s ago 22m 492M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (105s) 19s ago 20m 447M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (91s) 17s ago 23m 42.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (69s) 19s ago 20m 36.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (13m) 17s ago 13m 154M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (13m) 19s ago 13m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:03:27.599 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (6m) 17s ago 21m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (2m) 19s ago 20m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 starting - - - 4096M 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (19m) 17s ago 19m 487M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (19m) 17s ago 19m 355M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (18m) 17s ago 18m 482M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (18m) 19s ago 18m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (18m) 19s ago 18m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (17m) 19s ago 17m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (17m) 19s ago 17m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:03:27.600 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 17s ago 20m - - 2024-04-23T01:03:27.633 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:27 smithi121 ceph-mon[88653]: pgmap v50: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:27.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:27 smithi153 ceph-mon[65847]: pgmap v50: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 7 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:03:28.008 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 9, 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 4 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:03:28.009 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:03:28.344 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:03:28.345 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:03:28.345 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "6/23 daemons upgraded", 2024-04-23T01:03:28.345 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:03:28.345 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:03:28.345 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:03:28.747 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 67/438 objects degraded (15.297%), 18 pgs degraded 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: osd.0 (root=default,host=smithi121) is down 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 67/438 objects degraded (15.297%), 18 pgs degraded 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-23T01:03:28.748 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-04-23T01:03:28.749 INFO:teuthology.orchestra.run.smithi121.stdout: pg 5.5 is active+undersized+degraded, acting [1,4] 2024-04-23T01:03:28.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:28 smithi153 ceph-mon[65847]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:28 smithi153 ceph-mon[65847]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:28 smithi153 ceph-mon[65847]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:28 smithi153 ceph-mon[65847]: from='client.34192 172.21.15.121:0/2408463625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:28.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:28 smithi121 ceph-mon[88653]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:28 smithi121 ceph-mon[88653]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:28 smithi121 ceph-mon[88653]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:28.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:28 smithi121 ceph-mon[88653]: from='client.34192 172.21.15.121:0/2408463625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:29.695 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:29 smithi121 ceph-mon[88653]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:29.695 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:29 smithi121 ceph-mon[88653]: pgmap v51: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:29.695 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:29 smithi121 ceph-mon[88653]: from='client.34200 172.21.15.121:0/2551761030' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:03:29.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:29 smithi153 ceph-mon[65847]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:03:29.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:29 smithi153 ceph-mon[65847]: pgmap v51: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:29.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:29 smithi153 ceph-mon[65847]: from='client.34200 172.21.15.121:0/2551761030' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:03:31.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:31 smithi153 ceph-mon[65847]: pgmap v52: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:31.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:31 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:31.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:31 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:31.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:31 smithi121 ceph-mon[88653]: pgmap v52: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67/438 objects degraded (15.297%) 2024-04-23T01:03:31.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:31 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:31.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:31 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:32.383 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:32 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:03:32.124+0000 7fdbecf317c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-23T01:03:33.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:33.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:33.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:32 smithi121 ceph-mon[88653]: pgmap v53: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 67/438 objects degraded (15.297%) 2024-04-23T01:03:33.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:33.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:33.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:32 smithi153 ceph-mon[65847]: pgmap v53: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 67/438 objects degraded (15.297%) 2024-04-23T01:03:33.507 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:03:33.194+0000 7fdbecf317c0 -1 osd.0 76 log_to_monitors true 2024-04-23T01:03:34.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:33 smithi121 ceph-mon[88653]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-23T01:03:34.134 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:03:33 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:03:33.881+0000 7fdbde38b700 -1 osd.0 76 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:03:34.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:33 smithi153 ceph-mon[65847]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-23T01:03:35.203 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-23T01:03:35.204 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: osdmap e79: 8 total, 7 up, 8 in 2024-04-23T01:03:35.204 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:03:35.204 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: pgmap v55: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 67/438 objects degraded (15.297%) 2024-04-23T01:03:35.204 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:35.204 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:35.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-23T01:03:35.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: osdmap e79: 8 total, 7 up, 8 in 2024-04-23T01:03:35.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: from='osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:03:35.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: pgmap v55: 129 pgs: 18 active+undersized+degraded, 31 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 67/438 objects degraded (15.297%) 2024-04-23T01:03:35.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:35.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:35 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:03:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:35 smithi121 ceph-mon[88653]: osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182] boot 2024-04-23T01:03:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:35 smithi121 ceph-mon[88653]: osdmap e80: 8 total, 8 up, 8 in 2024-04-23T01:03:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:03:36.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:35 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:03:36.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:35 smithi153 ceph-mon[65847]: osd.0 [v2:172.21.15.121:6802/1135817182,v1:172.21.15.121:6803/1135817182] boot 2024-04-23T01:03:36.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:35 smithi153 ceph-mon[65847]: osdmap e80: 8 total, 8 up, 8 in 2024-04-23T01:03:36.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-23T01:03:37.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: osdmap e81: 8 total, 8 up, 8 in 2024-04-23T01:03:37.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: mgrmap e46: smithi121.otezso(active, since 91s), standbys: smithi153.olhmgd 2024-04-23T01:03:37.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: pgmap v58: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 348 B/s rd, 522 B/s wr, 0 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: pgmap v59: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 527 B/s rd, 790 B/s wr, 1 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:37.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:37.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:36 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-23T01:03:37.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: osdmap e81: 8 total, 8 up, 8 in 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: mgrmap e46: smithi121.otezso(active, since 91s), standbys: smithi153.olhmgd 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: pgmap v58: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 348 B/s rd, 522 B/s wr, 0 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: pgmap v59: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 527 B/s rd, 790 B/s wr, 1 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:03:37.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:37.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:37.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:36 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (17 PGs are or would become offline) 2024-04-23T01:03:38.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:37 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 12/438 objects degraded (2.740%), 3 pgs degraded (PG_DEGRADED) 2024-04-23T01:03:38.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:37 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 12/438 objects degraded (2.740%), 3 pgs degraded (PG_DEGRADED) 2024-04-23T01:03:39.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:38 smithi153 ceph-mon[65847]: pgmap v60: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 461 B/s rd, 692 B/s wr, 1 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:39.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:38 smithi121 ceph-mon[88653]: pgmap v60: 129 pgs: 33 peering, 3 active+undersized+degraded, 13 active+undersized, 80 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 461 B/s rd, 692 B/s wr, 1 op/s; 12/438 objects degraded (2.740%) 2024-04-23T01:03:40.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:40 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/438 objects degraded (2.740%), 3 pgs degraded) 2024-04-23T01:03:40.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:40 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/438 objects degraded (2.740%), 3 pgs degraded) 2024-04-23T01:03:41.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:41 smithi153 ceph-mon[65847]: pgmap v61: 129 pgs: 33 peering, 96 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 348 B/s rd, 522 B/s wr, 0 op/s 2024-04-23T01:03:41.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:41 smithi121 ceph-mon[88653]: pgmap v61: 129 pgs: 33 peering, 96 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 348 B/s rd, 522 B/s wr, 0 op/s 2024-04-23T01:03:43.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:43 smithi153 ceph-mon[65847]: pgmap v62: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 275 B/s rd, 413 B/s wr, 0 op/s 2024-04-23T01:03:43.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:43 smithi121 ceph-mon[88653]: pgmap v62: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 275 B/s rd, 413 B/s wr, 0 op/s 2024-04-23T01:03:45.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:45 smithi153 ceph-mon[65847]: pgmap v63: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:03:45.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:45 smithi121 ceph-mon[88653]: pgmap v63: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:03:47.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:47 smithi153 ceph-mon[65847]: pgmap v64: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:03:47.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:47 smithi121 ceph-mon[88653]: pgmap v64: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:03:49.829 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:49 smithi153 ceph-mon[65847]: pgmap v65: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:49.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:49 smithi121 ceph-mon[88653]: pgmap v65: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:51.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:51.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:51.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:51.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:51.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:03:51.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:51.960 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:51 smithi121 ceph-mon[88653]: pgmap v66: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:51.960 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:52.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:51 smithi153 ceph-mon[65847]: pgmap v66: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:52.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:52.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:52 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:52.883 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:52 smithi121 ceph-mon[88653]: Upgrade: osd.1 is safe to restart 2024-04-23T01:03:53.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:52 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-23T01:03:53.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:52 smithi153 ceph-mon[65847]: Upgrade: osd.1 is safe to restart 2024-04-23T01:03:53.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:53 smithi121 ceph-mon[88653]: pgmap v67: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:53.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:53.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T01:03:53.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:54.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:53 smithi153 ceph-mon[65847]: pgmap v67: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:54.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:03:54.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-23T01:03:54.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:03:54.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:54 smithi121 ceph-mon[88653]: Upgrade: Updating osd.1 2024-04-23T01:03:54.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:54 smithi121 ceph-mon[88653]: Deploying daemon osd.1 on smithi121 2024-04-23T01:03:55.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:54 smithi153 ceph-mon[65847]: Upgrade: Updating osd.1 2024-04-23T01:03:55.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:54 smithi153 ceph-mon[65847]: Deploying daemon osd.1 on smithi121 2024-04-23T01:03:55.854 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:55 smithi121 ceph-mon[88653]: pgmap v68: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:56.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:55 smithi153 ceph-mon[65847]: pgmap v68: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:03:56.442 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:56 smithi121 systemd[1]: Stopping Ceph osd.1 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:03:56.442 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T01:03:56.289+0000 7f5cb72d9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:03:56.443 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T01:03:56.289+0000 7f5cb72d9700 -1 osd.1 81 *** Got signal Terminated *** 2024-04-23T01:03:56.443 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1[49096]: debug 2024-04-23T01:03:56.289+0000 7f5cb72d9700 -1 osd.1 81 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:03:57.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.3 2024-04-23T01:03:57.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 failed (root=default,host=smithi121) (connection refused reported by osd.3) 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.2 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.3 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.6 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.0 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.7 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.4 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.2 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.5 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.6 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.0 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.7 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.4 2024-04-23T01:03:57.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:56 smithi153 ceph-mon[65847]: osd.1 reported immediately failed by osd.5 2024-04-23T01:03:57.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:56 smithi121 bash[98468]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.1 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.3 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 failed (root=default,host=smithi121) (connection refused reported by osd.3) 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.2 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.3 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.6 2024-04-23T01:03:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.0 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.7 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.4 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.2 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.5 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.6 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.0 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.7 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.4 2024-04-23T01:03:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:56 smithi121 ceph-mon[88653]: osd.1 reported immediately failed by osd.5 2024-04-23T01:03:57.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:57 smithi121 ceph-mon[88653]: pgmap v69: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:03:57.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:57 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:03:57.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:57 smithi121 ceph-mon[88653]: osdmap e82: 8 total, 7 up, 8 in 2024-04-23T01:03:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:57 smithi153 ceph-mon[65847]: pgmap v69: 129 pgs: 129 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:03:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:57 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:03:58.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:57 smithi153 ceph-mon[65847]: osdmap e82: 8 total, 7 up, 8 in 2024-04-23T01:03:58.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:58 smithi121 ceph-mon[88653]: osdmap e83: 8 total, 7 up, 8 in 2024-04-23T01:03:59.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:58 smithi153 ceph-mon[65847]: osdmap e83: 8 total, 7 up, 8 in 2024-04-23T01:03:59.107 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:03:59.767 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:03:59.767 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (11m) 28s ago 22m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:03:59.767 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (61s) 28s ago 22m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (56s) 51s ago 21m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (9m) 28s ago 21m 34.6M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (16m) 28s ago 16m 15.7M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (16m) 51s ago 16m 42.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (2m) 28s ago 23m 495M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (2m) 51s ago 21m 447M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (2m) 28s ago 23m 49.3M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (101s) 51s ago 21m 36.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (13m) 28s ago 13m 154M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (13m) 51s ago 13m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (7m) 28s ago 21m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (2m) 51s ago 20m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (35s) 28s ago 20m 13.5M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:03:59.768 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (20m) 28s ago 20m 486M 4096M 16.2.0 24ecd6d5f14c 201f8e727bf3 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (19m) 28s ago 19m 353M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (19m) 28s ago 19m 479M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (19m) 51s ago 19m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (18m) 51s ago 18m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (18m) 51s ago 18m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (18m) 51s ago 17m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:03:59.769 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 28s ago 21m - - 2024-04-23T01:04:00.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:03:59 smithi153 ceph-mon[65847]: pgmap v72: 129 pgs: 18 stale+active+clean, 111 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:04:00.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:03:59 smithi121 ceph-mon[88653]: pgmap v72: 129 pgs: 18 stale+active+clean, 111 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:04:00.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:59 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.1.service: Succeeded. 2024-04-23T01:04:00.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:59 smithi121 systemd[1]: Stopped Ceph osd.1 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:04:00.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:03:59 smithi121 systemd[1]: Starting Ceph osd.1 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:04:00.185 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 6, 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 1 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8, 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 5 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:04:00.186 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:04:00.519 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:04:00.519 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:04:00.519 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:04:00.519 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "7/23 daemons upgraded", 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:04:00.520 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 5/438 objects degraded (1.142%), 2 pgs degraded 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout: osd.1 (root=default,host=smithi121) is down 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 5/438 objects degraded (1.142%), 2 pgs degraded 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-04-23T01:04:00.926 INFO:teuthology.orchestra.run.smithi121.stdout: pg 5.5 is active+undersized+degraded, acting [0,4] 2024-04-23T01:04:01.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:00 smithi153 ceph-mon[65847]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:00 smithi153 ceph-mon[65847]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:00 smithi153 ceph-mon[65847]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:00 smithi153 ceph-mon[65847]: from='client.34218 172.21.15.121:0/3979798765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:00 smithi121 ceph-mon[88653]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:00 smithi121 ceph-mon[88653]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:00 smithi121 ceph-mon[88653]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:00 smithi121 ceph-mon[88653]: from='client.34218 172.21.15.121:0/3979798765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:01.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-23T01:04:01.134 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-23T01:04:01.135 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-23T01:04:01.135 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-23T01:04:01.135 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-23T01:04:01.135 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-23T01:04:01.135 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:00 smithi121 bash[98959]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-23T01:04:02.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:01 smithi153 ceph-mon[65847]: pgmap v73: 129 pgs: 2 active+undersized+degraded, 17 stale+active+clean, 3 active+undersized, 107 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/438 objects degraded (1.142%) 2024-04-23T01:04:02.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:01 smithi153 ceph-mon[65847]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:02.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:01 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 5/438 objects degraded (1.142%), 2 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:02.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:01 smithi153 ceph-mon[65847]: from='client.34226 172.21.15.121:0/1887373849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:04:02.111 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:01 smithi121 ceph-mon[88653]: pgmap v73: 129 pgs: 2 active+undersized+degraded, 17 stale+active+clean, 3 active+undersized, 107 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/438 objects degraded (1.142%) 2024-04-23T01:04:02.112 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:01 smithi121 ceph-mon[88653]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:02.112 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:01 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 5/438 objects degraded (1.142%), 2 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:02.112 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:01 smithi121 ceph-mon[88653]: from='client.34226 172.21.15.121:0/1887373849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:04:02.384 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:02 smithi121 ceph-osd[99301]: -- 172.21.15.121:0/1009893604 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d1a1b511e0 con 0x55d1a1b20c00 2024-04-23T01:04:02.384 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:02 smithi121 bash[99263]: 0ebdf9aa6cdfea68af9a7ea506a783b282ceeb6c5a8e7b166b9c1a6309eabd2e 2024-04-23T01:04:02.384 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:02 smithi121 systemd[1]: Started Ceph osd.1 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:04:02.992 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:02 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:04:02.626+0000 7f54c6c567c0 -1 Falling back to public interface 2024-04-23T01:04:03.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:03 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:03.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:03 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:03.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:03 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:03.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:03 smithi121 ceph-mon[88653]: pgmap v74: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:03.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:03 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:03.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:03 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:03.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:03 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:03.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:03 smithi153 ceph-mon[65847]: pgmap v74: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:05.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:05 smithi121 ceph-mon[88653]: pgmap v75: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:05.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:05.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:05 smithi153 ceph-mon[65847]: pgmap v75: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:06.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:07.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:06 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 49/438 objects degraded (11.187%), 17 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:07.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:06 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 49/438 objects degraded (11.187%), 17 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:08.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:07 smithi153 ceph-mon[65847]: pgmap v76: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:08.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:07 smithi121 ceph-mon[88653]: pgmap v76: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:08.977 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:04:08.915+0000 7f54c6c567c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-23T01:04:09.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-mon[88653]: pgmap v77: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:09.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.366 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:09 smithi153 ceph-mon[65847]: pgmap v77: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:09.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:10.126 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:09 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:04:09.800+0000 7f54c6c567c0 -1 osd.1 81 log_to_monitors true 2024-04-23T01:04:10.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:10 smithi121 ceph-mon[88653]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-23T01:04:10.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:10 smithi153 ceph-mon[65847]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-23T01:04:11.094 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:04:10 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:04:10.644+0000 7f54b40a8700 -1 osd.1 81 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:04:11.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:11 smithi121 ceph-mon[88653]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-23T01:04:11.375 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:11 smithi121 ceph-mon[88653]: osdmap e84: 8 total, 7 up, 8 in 2024-04-23T01:04:11.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:11 smithi121 ceph-mon[88653]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:04:11.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:11 smithi121 ceph-mon[88653]: pgmap v79: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:11.376 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:11 smithi121 ceph-mon[88653]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' 2024-04-23T01:04:11.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:11 smithi153 ceph-mon[65847]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-23T01:04:11.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:11 smithi153 ceph-mon[65847]: osdmap e84: 8 total, 7 up, 8 in 2024-04-23T01:04:11.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:11 smithi153 ceph-mon[65847]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:04:11.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:11 smithi153 ceph-mon[65847]: pgmap v79: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:11.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:11 smithi153 ceph-mon[65847]: from='osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268]' entity='osd.1' 2024-04-23T01:04:12.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:12 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:04:12.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:12 smithi153 ceph-mon[65847]: osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268] boot 2024-04-23T01:04:12.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:12 smithi153 ceph-mon[65847]: osdmap e85: 8 total, 8 up, 8 in 2024-04-23T01:04:12.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:12 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:04:12.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:12 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:04:12.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:12 smithi121 ceph-mon[88653]: osd.1 [v2:172.21.15.121:6810/451517268,v1:172.21.15.121:6811/451517268] boot 2024-04-23T01:04:12.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:12 smithi121 ceph-mon[88653]: osdmap e85: 8 total, 8 up, 8 in 2024-04-23T01:04:12.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:12 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-23T01:04:14.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: pgmap v81: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: osdmap e86: 8 total, 8 up, 8 in 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:14.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: pgmap v81: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:04:14.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: osdmap e86: 8 total, 8 up, 8 in 2024-04-23T01:04:14.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:14 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:04:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:14 smithi153 ceph-mon[65847]: pgmap v83: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:14 smithi153 ceph-mon[65847]: pgmap v84: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:14 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:14 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-23T01:04:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:14 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:04:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:14 smithi121 ceph-mon[88653]: pgmap v83: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:14 smithi121 ceph-mon[88653]: pgmap v84: 129 pgs: 17 active+undersized+degraded, 36 active+undersized, 76 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 49/438 objects degraded (11.187%) 2024-04-23T01:04:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:14 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:14 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-23T01:04:16.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:15 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:16.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:15 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:17.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:16 smithi153 ceph-mon[65847]: pgmap v85: 129 pgs: 16 active+undersized+degraded, 33 active+undersized, 80 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:04:17.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:16 smithi121 ceph-mon[88653]: pgmap v85: 129 pgs: 16 active+undersized+degraded, 33 active+undersized, 80 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:04:18.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:17 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded) 2024-04-23T01:04:18.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:17 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded) 2024-04-23T01:04:19.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:18 smithi153 ceph-mon[65847]: pgmap v86: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 375 B/s rd, 0 op/s 2024-04-23T01:04:19.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:18 smithi121 ceph-mon[88653]: pgmap v86: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 375 B/s rd, 0 op/s 2024-04-23T01:04:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:20 smithi121 ceph-mon[88653]: pgmap v87: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-04-23T01:04:20.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:20.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:21.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:20 smithi153 ceph-mon[65847]: pgmap v87: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 302 B/s rd, 0 op/s 2024-04-23T01:04:21.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:23.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:22 smithi153 ceph-mon[65847]: pgmap v88: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 242 B/s rd, 0 op/s 2024-04-23T01:04:23.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:22 smithi121 ceph-mon[88653]: pgmap v88: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 242 B/s rd, 0 op/s 2024-04-23T01:04:25.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:24 smithi153 ceph-mon[65847]: pgmap v89: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:04:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:24 smithi121 ceph-mon[88653]: pgmap v89: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:04:26.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:26.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:27.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:26 smithi153 ceph-mon[65847]: pgmap v90: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:27.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:26 smithi121 ceph-mon[88653]: pgmap v90: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:29.064 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:28 smithi121 ceph-mon[88653]: pgmap v91: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:04:29.064 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:29.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:28 smithi153 ceph-mon[65847]: pgmap v91: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:04:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:30.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:29 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:30.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:29 smithi153 ceph-mon[65847]: Upgrade: osd.2 is safe to restart 2024-04-23T01:04:30.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:29 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-23T01:04:30.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:29 smithi121 ceph-mon[88653]: Upgrade: osd.2 is safe to restart 2024-04-23T01:04:31.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:30 smithi121 ceph-mon[88653]: pgmap v92: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:31.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:31.034 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T01:04:31.035 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:31.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:30 smithi153 ceph-mon[65847]: pgmap v92: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:31.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:31.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-23T01:04:31.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:31.267 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:04:31.937 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:31 smithi121 ceph-mon[88653]: Upgrade: Updating osd.2 2024-04-23T01:04:31.937 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:31 smithi121 ceph-mon[88653]: Deploying daemon osd.2 on smithi121 2024-04-23T01:04:31.955 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:04:31.955 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (12m) 23s ago 22m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (93s) 23s ago 22m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (88s) 84s ago 21m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (9m) 23s ago 22m 34.6M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (17m) 23s ago 16m 15.9M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (17m) 84s ago 17m 42.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (3m) 23s ago 24m 497M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (2m) 84s ago 21m 447M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (2m) 23s ago 24m 52.2M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (2m) 84s ago 21m 36.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (14m) 23s ago 14m 157M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (14m) 84s ago 14m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (7m) 23s ago 22m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (3m) 84s ago 21m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (67s) 23s ago 20m 207M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:04:31.956 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (30s) 23s ago 20m 11.4M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (20m) 23s ago 20m 353M 4096M 16.2.0 24ecd6d5f14c c102a47c539c 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (19m) 23s ago 19m 480M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (19m) 84s ago 19m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (19m) 84s ago 19m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (18m) 84s ago 18m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (18m) 84s ago 18m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:04:31.957 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 23s ago 21m - - 2024-04-23T01:04:32.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:31 smithi153 ceph-mon[65847]: Upgrade: Updating osd.2 2024-04-23T01:04:32.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:31 smithi153 ceph-mon[65847]: Deploying daemon osd.2 on smithi121 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 6, 2024-04-23T01:04:32.367 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8, 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 6 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:04:32.368 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:04:32.695 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "8/23 daemons upgraded", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:04:32.696 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:04:32.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:32 smithi121 ceph-mon[88653]: pgmap v93: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:32.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:32 smithi121 ceph-mon[88653]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:32.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:32 smithi121 ceph-mon[88653]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:32.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:32 smithi121 ceph-mon[88653]: from='client.34246 172.21.15.121:0/1688418516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:33.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:32 smithi153 ceph-mon[65847]: pgmap v93: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:33.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:32 smithi153 ceph-mon[65847]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:33.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:32 smithi153 ceph-mon[65847]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:33.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:32 smithi153 ceph-mon[65847]: from='client.34246 172.21.15.121:0/1688418516' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:33.098 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:04:33.098 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:04:33.098 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:04:33.995 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:33 smithi121 ceph-mon[88653]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:33.996 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:33 smithi121 ceph-mon[88653]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:33.996 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:33 smithi121 ceph-mon[88653]: from='client.34254 172.21.15.121:0/1646389750' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:04:34.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:33 smithi153 ceph-mon[65847]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:34.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:33 smithi153 ceph-mon[65847]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:04:34.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:33 smithi153 ceph-mon[65847]: from='client.34254 172.21.15.121:0/1646389750' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:04:34.385 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:33 smithi121 systemd[1]: Stopping Ceph osd.2 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:04:34.385 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T01:04:34.112+0000 7f3e4f817700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:04:34.385 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T01:04:34.112+0000 7f3e4f817700 -1 osd.2 86 *** Got signal Terminated *** 2024-04-23T01:04:34.385 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2[52760]: debug 2024-04-23T01:04:34.112+0000 7f3e4f817700 -1 osd.2 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:04:34.884 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:34 smithi121 bash[102123]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.2 2024-04-23T01:04:34.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: pgmap v94: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.1 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 failed (root=default,host=smithi121) (connection refused reported by osd.1) 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.1 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.3 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.0 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.7 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.4 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.5 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.3 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.6 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.0 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.7 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.4 2024-04-23T01:04:34.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.5 2024-04-23T01:04:34.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: osd.2 reported immediately failed by osd.6 2024-04-23T01:04:34.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:35.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: pgmap v94: 129 pgs: 129 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.1 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 failed (root=default,host=smithi121) (connection refused reported by osd.1) 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.1 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.3 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.0 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.7 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.4 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.5 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.3 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.6 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.0 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.7 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.4 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.5 2024-04-23T01:04:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: osd.2 reported immediately failed by osd.6 2024-04-23T01:04:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:35 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:04:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:35 smithi153 ceph-mon[65847]: osdmap e87: 8 total, 7 up, 8 in 2024-04-23T01:04:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:35 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:04:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:35 smithi121 ceph-mon[88653]: osdmap e87: 8 total, 7 up, 8 in 2024-04-23T01:04:37.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:36 smithi153 ceph-mon[65847]: pgmap v96: 129 pgs: 10 stale+active+clean, 119 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:04:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:36 smithi153 ceph-mon[65847]: osdmap e88: 8 total, 7 up, 8 in 2024-04-23T01:04:37.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:36 smithi121 ceph-mon[88653]: pgmap v96: 129 pgs: 10 stale+active+clean, 119 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:04:37.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:36 smithi121 ceph-mon[88653]: osdmap e88: 8 total, 7 up, 8 in 2024-04-23T01:04:37.135 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:36 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.2.service: Succeeded. 2024-04-23T01:04:37.135 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:36 smithi121 systemd[1]: Stopped Ceph osd.2 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:04:37.135 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:36 smithi121 systemd[1]: Starting Ceph osd.2 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:04:38.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:37 smithi153 ceph-mon[65847]: Health check failed: Reduced data availability: 2 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-23T01:04:38.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:37 smithi121 ceph-mon[88653]: Health check failed: Reduced data availability: 2 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-23T01:04:38.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:37 smithi121 bash[102422]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-23T01:04:39.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:38 smithi121 ceph-mon[88653]: pgmap v98: 129 pgs: 30 peering, 2 stale+active+clean, 97 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:39.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:38 smithi121 ceph-osd[102744]: -- 172.21.15.121:0/3861235398 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56322bdf51e0 con 0x56322bdc4c00 2024-04-23T01:04:39.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:39 smithi121 bash[102707]: c20ed2b44a08e08574657cfd213c6b66494e74947f8d61ec4ff48694d31f548d 2024-04-23T01:04:39.134 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:39 smithi121 systemd[1]: Started Ceph osd.2 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:04:39.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:38 smithi153 ceph-mon[65847]: pgmap v98: 129 pgs: 30 peering, 2 stale+active+clean, 97 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:39.811 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:04:39.462+0000 7f3c3f8ec7c0 -1 Falling back to public interface 2024-04-23T01:04:40.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:40 smithi153 ceph-mon[65847]: pgmap v99: 129 pgs: 30 peering, 5 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:40.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:40.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:40.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:40 smithi121 ceph-mon[88653]: pgmap v99: 129 pgs: 30 peering, 5 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:40.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:41.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:41 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 45/438 objects degraded (10.274%), 13 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:41.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:41 smithi153 ceph-mon[65847]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 8 pgs peering) 2024-04-23T01:04:41.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:41 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 45/438 objects degraded (10.274%), 13 pgs degraded (PG_DEGRADED) 2024-04-23T01:04:41.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:41 smithi121 ceph-mon[88653]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 8 pgs peering) 2024-04-23T01:04:42.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:42 smithi121 ceph-mon[88653]: pgmap v100: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:42.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:42 smithi153 ceph-mon[65847]: pgmap v100: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:44.368 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:44 smithi121 ceph-mon[88653]: pgmap v101: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:44.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:44 smithi153 ceph-mon[65847]: pgmap v101: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:45.109 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:44 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:04:44.762+0000 7f3c3f8ec7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-23T01:04:45.884 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:45 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:04:45.562+0000 7f3c3f8ec7c0 -1 osd.2 86 log_to_monitors true 2024-04-23T01:04:46.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:46 smithi121 ceph-mon[88653]: pgmap v102: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:46.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:46 smithi121 ceph-mon[88653]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-23T01:04:46.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:46 smithi153 ceph-mon[65847]: pgmap v102: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:46.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:46 smithi153 ceph-mon[65847]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-23T01:04:47.135 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:04:46 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:04:46.684+0000 7f3c2cd3e700 -1 osd.2 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: osdmap e89: 8 total, 7 up, 8 in 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:47 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: osdmap e89: 8 total, 7 up, 8 in 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209]' entity='osd.2' 2024-04-23T01:04:47.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:47.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:47 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:48.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:48 smithi121 ceph-mon[88653]: pgmap v104: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:48.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:48 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:04:48.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:48 smithi121 ceph-mon[88653]: osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209] boot 2024-04-23T01:04:48.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:48 smithi121 ceph-mon[88653]: osdmap e90: 8 total, 8 up, 8 in 2024-04-23T01:04:48.346 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:04:48.579 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:48 smithi153 ceph-mon[65847]: pgmap v104: 129 pgs: 13 active+undersized+degraded, 22 active+undersized, 94 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:48.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:48 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:04:48.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:48 smithi153 ceph-mon[65847]: osd.2 [v2:172.21.15.121:6818/999539209,v1:172.21.15.121:6819/999539209] boot 2024-04-23T01:04:48.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:48 smithi153 ceph-mon[65847]: osdmap e90: 8 total, 8 up, 8 in 2024-04-23T01:04:48.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-23T01:04:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:49 smithi153 ceph-mon[65847]: osdmap e91: 8 total, 8 up, 8 in 2024-04-23T01:04:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:50.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:49 smithi121 ceph-mon[88653]: osdmap e91: 8 total, 8 up, 8 in 2024-04-23T01:04:50.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:50.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:04:50.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: pgmap v107: 129 pgs: 13 active+undersized+degraded, 19 active+undersized, 97 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:50.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:50.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:50.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:50.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:04:50.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:50.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: pgmap v107: 129 pgs: 13 active+undersized+degraded, 19 active+undersized, 97 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: pgmap v108: 129 pgs: 13 active+undersized+degraded, 15 active+undersized, 101 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 372 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: pgmap v109: 129 pgs: 13 active+undersized+degraded, 15 active+undersized, 101 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:04:52.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:51 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-23T01:04:52.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:04:52.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: pgmap v108: 129 pgs: 13 active+undersized+degraded, 15 active+undersized, 101 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 372 B/s rd, 0 op/s; 45/438 objects degraded (10.274%) 2024-04-23T01:04:52.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: pgmap v109: 129 pgs: 13 active+undersized+degraded, 15 active+undersized, 101 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:04:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:51 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-23T01:04:53.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:52 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/438 objects degraded (10.274%), 13 pgs degraded) 2024-04-23T01:04:53.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:52 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/438 objects degraded (10.274%), 13 pgs degraded) 2024-04-23T01:04:54.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:53 smithi153 ceph-mon[65847]: pgmap v110: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:54.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:53 smithi121 ceph-mon[88653]: pgmap v110: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:56.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:55 smithi153 ceph-mon[65847]: pgmap v111: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:56.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:56.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:55 smithi121 ceph-mon[88653]: pgmap v111: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-04-23T01:04:56.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:04:58.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:57 smithi153 ceph-mon[65847]: pgmap v112: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 258 B/s rd, 0 op/s 2024-04-23T01:04:58.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:57 smithi121 ceph-mon[88653]: pgmap v112: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 258 B/s rd, 0 op/s 2024-04-23T01:05:00.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:04:59 smithi153 ceph-mon[65847]: pgmap v113: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-04-23T01:05:00.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:04:59 smithi121 ceph-mon[88653]: pgmap v113: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-04-23T01:05:02.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:01 smithi153 ceph-mon[65847]: pgmap v114: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:05:02.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:01 smithi121 ceph-mon[88653]: pgmap v114: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:05:03.446 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:05:04.072 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:05:04.072 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (12m) 17s ago 23m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:05:04.072 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (2m) 17s ago 23m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:05:04.072 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (2m) 116s ago 22m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:05:04.072 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (10m) 17s ago 22m 34.6M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (17m) 17s ago 17m 16.0M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (17m) 116s ago 17m 42.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (3m) 17s ago 24m 498M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (3m) 116s ago 22m 447M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (3m) 17s ago 24m 53.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (2m) 116s ago 22m 36.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (14m) 17s ago 14m 183M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (14m) 116s ago 14m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (8m) 17s ago 22m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (3m) 116s ago 21m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (99s) 17s ago 21m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (62s) 17s ago 21m 191M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (25s) 17s ago 20m 13.7M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:05:04.073 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (20m) 17s ago 20m 479M 4096M 16.2.0 24ecd6d5f14c ff599656b069 2024-04-23T01:05:04.074 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (20m) 116s ago 20m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:05:04.074 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (19m) 116s ago 19m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:05:04.074 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (19m) 116s ago 19m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:05:04.074 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (19m) 116s ago 19m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:05:04.074 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 17s ago 22m - - de242295e225 7fa20fb9728e 2024-04-23T01:05:04.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:03 smithi153 ceph-mon[65847]: pgmap v115: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:04.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:03 smithi121 ceph-mon[88653]: pgmap v115: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:05:04.453 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 5, 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 3 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 7, 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 7 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:05:04.454 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:05:04.785 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:05:04.785 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:05:04.785 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:05:04.785 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "9/23 daemons upgraded", 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:05:04.786 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:05:05.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:04 smithi153 ceph-mon[65847]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:05.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:04 smithi153 ceph-mon[65847]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:05.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:04 smithi153 ceph-mon[65847]: from='client.34274 172.21.15.121:0/2435053299' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:05.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:05.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:05.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:04 smithi121 ceph-mon[88653]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:05.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:04 smithi121 ceph-mon[88653]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:05.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:04 smithi121 ceph-mon[88653]: from='client.34274 172.21.15.121:0/2435053299' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:05.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:05.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:05.166 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:05:05.166 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:05:05.167 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:05:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:05 smithi153 ceph-mon[65847]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:05 smithi153 ceph-mon[65847]: pgmap v116: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:05 smithi153 ceph-mon[65847]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:05 smithi153 ceph-mon[65847]: from='client.34282 172.21.15.121:0/1811258002' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:05:06.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:05 smithi121 ceph-mon[88653]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:06.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:05 smithi121 ceph-mon[88653]: pgmap v116: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:06.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:05 smithi121 ceph-mon[88653]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:06.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:05 smithi121 ceph-mon[88653]: from='client.34282 172.21.15.121:0/1811258002' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:05:07.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:06 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:05:07.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:06 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:05:07.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:06 smithi153 ceph-mon[65847]: Upgrade: osd.3 is safe to restart 2024-04-23T01:05:07.109 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:06 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:05:07.109 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:06 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-23T01:05:07.109 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:06 smithi121 ceph-mon[88653]: Upgrade: osd.3 is safe to restart 2024-04-23T01:05:07.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:07 smithi121 ceph-mon[88653]: pgmap v117: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:05:08.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:07 smithi153 ceph-mon[65847]: pgmap v117: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: Upgrade: Updating osd.3 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: Deploying daemon osd.3 on smithi121 2024-04-23T01:05:09.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:09 smithi153 ceph-mon[65847]: pgmap v118: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:09.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: Upgrade: Updating osd.3 2024-04-23T01:05:09.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:09.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-23T01:05:09.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:09.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: Deploying daemon osd.3 on smithi121 2024-04-23T01:05:09.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:09 smithi121 ceph-mon[88653]: pgmap v118: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:11.635 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:11 smithi121 systemd[1]: Stopping Ceph osd.3 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:05:11.635 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T01:05:11.524+0000 7fe8c8c03700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:05:11.635 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T01:05:11.524+0000 7fe8c8c03700 -1 osd.3 91 *** Got signal Terminated *** 2024-04-23T01:05:11.636 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3[56123]: debug 2024-04-23T01:05:11.524+0000 7fe8c8c03700 -1 osd.3 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:05:11.955 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:11 smithi121 ceph-mon[88653]: pgmap v119: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:12.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:11 smithi153 ceph-mon[65847]: pgmap v119: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:12.256 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:12 smithi121 bash[105546]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.3 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 failed (root=default,host=smithi121) (connection refused reported by osd.5) 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.049 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.050 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.051 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.052 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:12 smithi121 ceph-mon[88653]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 failed (root=default,host=smithi121) (connection refused reported by osd.5) 2024-04-23T01:05:13.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.1 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.2 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.6 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.5 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.7 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.0 2024-04-23T01:05:13.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:12 smithi153 ceph-mon[65847]: osd.3 reported immediately failed by osd.4 2024-04-23T01:05:13.854 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:13 smithi121 ceph-mon[88653]: pgmap v120: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:13.854 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:13 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:05:13.854 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:13 smithi121 ceph-mon[88653]: osdmap e92: 8 total, 7 up, 8 in 2024-04-23T01:05:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:13 smithi153 ceph-mon[65847]: pgmap v120: 129 pgs: 129 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:13 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:05:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:13 smithi153 ceph-mon[65847]: osdmap e92: 8 total, 7 up, 8 in 2024-04-23T01:05:14.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:14 smithi121 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.3.service: Succeeded. 2024-04-23T01:05:14.635 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:14 smithi121 systemd[1]: Stopped Ceph osd.3 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:05:14.635 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:14 smithi121 systemd[1]: Starting Ceph osd.3 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:05:15.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:14 smithi153 ceph-mon[65847]: osdmap e93: 8 total, 7 up, 8 in 2024-04-23T01:05:15.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:14 smithi121 ceph-mon[88653]: osdmap e93: 8 total, 7 up, 8 in 2024-04-23T01:05:15.622 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-23T01:05:15.623 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:15 smithi121 bash[105872]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-23T01:05:15.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:15 smithi121 ceph-mon[88653]: pgmap v123: 129 pgs: 2 active+undersized+degraded, 16 stale+active+clean, 8 active+undersized, 103 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9/438 objects degraded (2.055%) 2024-04-23T01:05:15.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:15 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 9/438 objects degraded (2.055%), 2 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:16.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:15 smithi153 ceph-mon[65847]: pgmap v123: 129 pgs: 2 active+undersized+degraded, 16 stale+active+clean, 8 active+undersized, 103 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9/438 objects degraded (2.055%) 2024-04-23T01:05:16.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:15 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 9/438 objects degraded (2.055%), 2 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:16.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:16 smithi121 ceph-osd[106173]: -- 172.21.15.121:0/1390125044 <== mon.0 v2:172.21.15.121:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5594bc9d51e0 con 0x5594bc9a4c00 2024-04-23T01:05:16.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:16 smithi121 bash[106135]: 3958601e8366300c37218ce7c6798b5d36ef70efabe35c8ec7f7ce842aadefec 2024-04-23T01:05:16.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:16 smithi121 systemd[1]: Started Ceph osd.3 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:05:17.220 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:16 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:05:16.885+0000 7fe65fcfc7c0 -1 Falling back to public interface 2024-04-23T01:05:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:17 smithi153 ceph-mon[65847]: pgmap v124: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:17.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:17.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:17.884 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:17.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:17 smithi121 ceph-mon[88653]: pgmap v124: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:19.943 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:19 smithi121 ceph-mon[88653]: pgmap v125: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:20.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:19 smithi153 ceph-mon[65847]: pgmap v125: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:21.047 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:21.047 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:21.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:21.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:21 smithi121 ceph-mon[88653]: pgmap v126: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:21.976 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:21 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:22.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:21 smithi153 ceph-mon[65847]: pgmap v126: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:22.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:21 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:23.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:23 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:05:23.197+0000 7fe65fcfc7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-23T01:05:24.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:23 smithi153 ceph-mon[65847]: pgmap v127: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:24.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:24.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:24.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:23 smithi121 ceph-mon[88653]: pgmap v127: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:24.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:24.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:24.634 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:24 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:05:24.325+0000 7fe65fcfc7c0 -1 osd.3 91 log_to_monitors true 2024-04-23T01:05:25.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:24 smithi153 ceph-mon[65847]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-23T01:05:25.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:24 smithi121 ceph-mon[88653]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-23T01:05:25.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:25.558 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:05:25.284+0000 7fe64d14e700 -1 osd.3 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:05:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:25 smithi153 ceph-mon[65847]: pgmap v128: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:25 smithi153 ceph-mon[65847]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-23T01:05:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:25 smithi153 ceph-mon[65847]: osdmap e94: 8 total, 7 up, 8 in 2024-04-23T01:05:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:25 smithi153 ceph-mon[65847]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:05:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:25 smithi153 ceph-mon[65847]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' 2024-04-23T01:05:26.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-mon[88653]: pgmap v128: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:05:26.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-mon[88653]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-23T01:05:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-mon[88653]: osdmap e94: 8 total, 7 up, 8 in 2024-04-23T01:05:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-mon[88653]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi121", "root=default"]}]: dispatch 2024-04-23T01:05:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:25 smithi121 ceph-mon[88653]: from='osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363]' entity='osd.3' 2024-04-23T01:05:27.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:26 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:05:27.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:26 smithi153 ceph-mon[65847]: osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363] boot 2024-04-23T01:05:27.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:26 smithi153 ceph-mon[65847]: osdmap e95: 8 total, 8 up, 8 in 2024-04-23T01:05:27.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:26 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:05:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:26 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:05:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:26 smithi121 ceph-mon[88653]: osd.3 [v2:172.21.15.121:6826/2688207363,v1:172.21.15.121:6827/2688207363] boot 2024-04-23T01:05:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:26 smithi121 ceph-mon[88653]: osdmap e95: 8 total, 8 up, 8 in 2024-04-23T01:05:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:26 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: pgmap v131: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: osdmap e96: 8 total, 8 up, 8 in 2024-04-23T01:05:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:27 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:28.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: pgmap v131: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: osdmap e96: 8 total, 8 up, 8 in 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:27 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:28 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:05:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:28 smithi153 ceph-mon[65847]: pgmap v132: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:28 smithi153 ceph-mon[65847]: pgmap v133: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 441 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:28 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:29.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:28 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-23T01:05:29.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:28 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:05:29.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:28 smithi121 ceph-mon[88653]: pgmap v132: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:28 smithi121 ceph-mon[88653]: pgmap v133: 129 pgs: 16 active+undersized+degraded, 34 active+undersized, 79 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 441 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:05:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:28 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:28 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-04-23T01:05:30.069 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:29 smithi121 ceph-mon[88653]: pgmap v135: 129 pgs: 15 active+undersized+degraded, 29 active+undersized, 85 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 462 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:05:30.069 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:29 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 41/438 objects degraded (9.361%), 15 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:30.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:29 smithi153 ceph-mon[65847]: pgmap v135: 129 pgs: 15 active+undersized+degraded, 29 active+undersized, 85 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 462 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:05:30.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:29 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 41/438 objects degraded (9.361%), 15 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:32.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:32 smithi153 ceph-mon[65847]: pgmap v136: 129 pgs: 3 active+undersized+degraded, 1 active+undersized, 125 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 34 op/s; 10/438 objects degraded (2.283%) 2024-04-23T01:05:32.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:32 smithi121 ceph-mon[88653]: pgmap v136: 129 pgs: 3 active+undersized+degraded, 1 active+undersized, 125 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 34 op/s; 10/438 objects degraded (2.283%) 2024-04-23T01:05:33.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:33 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/438 objects degraded (2.283%), 3 pgs degraded) 2024-04-23T01:05:33.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:33 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/438 objects degraded (2.283%), 3 pgs degraded) 2024-04-23T01:05:34.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:34 smithi121 ceph-mon[88653]: pgmap v137: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 25 op/s 2024-04-23T01:05:34.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:34 smithi153 ceph-mon[65847]: pgmap v137: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 25 op/s 2024-04-23T01:05:35.520 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:05:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:36.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:36.150 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:05:36.150 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (13m) 12s ago 23m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (2m) 12s ago 23m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (2m) 2m ago 22m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (10m) 12s ago 23m 34.3M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (18m) 12s ago 18m 16.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (18m) 2m ago 18m 42.4M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (4m) 12s ago 25m 499M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (3m) 2m ago 22m 447M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (3m) 12s ago 25m 55.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (3m) 2m ago 22m 36.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (15m) 12s ago 15m 192M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (15m) 2m ago 15m 84.8M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (8m) 12s ago 23m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:05:36.151 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (4m) 2m ago 22m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (2m) 12s ago 22m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (94s) 12s ago 21m 192M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (57s) 12s ago 21m 176M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (20s) 12s ago 20m 13.6M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (20m) 2m ago 20m 491M 4096M 16.2.0 24ecd6d5f14c 62349feaab95 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (20m) 2m ago 20m 595M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (19m) 2m ago 19m 509M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (19m) 2m ago 19m 550M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:05:36.152 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 12s ago 22m - - 2024-04-23T01:05:36.533 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 4, 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 4 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:05:36.534 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:05:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:05:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 6, 2024-04-23T01:05:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:05:36.535 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:05:36.535 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:05:36.859 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:05:36.860 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "10/23 daemons upgraded", 2024-04-23T01:05:36.860 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:05:36.860 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:05:36.860 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:05:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:36 smithi153 ceph-mon[65847]: pgmap v138: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 21 op/s 2024-04-23T01:05:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:36 smithi153 ceph-mon[65847]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:36 smithi153 ceph-mon[65847]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:36 smithi153 ceph-mon[65847]: from='client.34302 172.21.15.121:0/841489648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:37.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:36 smithi121 ceph-mon[88653]: pgmap v138: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 21 op/s 2024-04-23T01:05:37.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:36 smithi121 ceph-mon[88653]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:37.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:36 smithi121 ceph-mon[88653]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:37.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:36 smithi121 ceph-mon[88653]: from='client.34302 172.21.15.121:0/841489648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:05:37.239 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:05:37.239 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:05:37.239 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:05:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:37 smithi153 ceph-mon[65847]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:37 smithi153 ceph-mon[65847]: from='client.34310 172.21.15.121:0/2757733221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:05:38.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:37 smithi121 ceph-mon[88653]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:38.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:37 smithi121 ceph-mon[88653]: from='client.34310 172.21.15.121:0/2757733221' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:05:39.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:38 smithi153 ceph-mon[65847]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:39.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:38 smithi153 ceph-mon[65847]: pgmap v139: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 17 op/s 2024-04-23T01:05:39.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:38 smithi121 ceph-mon[88653]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:05:39.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:38 smithi121 ceph-mon[88653]: pgmap v139: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 17 op/s 2024-04-23T01:05:40.964 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:40 smithi121 ceph-mon[88653]: pgmap v140: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-04-23T01:05:41.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:40 smithi153 ceph-mon[65847]: pgmap v140: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-04-23T01:05:43.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:42 smithi153 ceph-mon[65847]: pgmap v141: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-04-23T01:05:43.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:43.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:42 smithi121 ceph-mon[88653]: pgmap v141: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-04-23T01:05:43.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:44.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:43 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:44.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:43 smithi153 ceph-mon[65847]: Upgrade: osd.4 is safe to restart 2024-04-23T01:05:44.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:43 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-23T01:05:44.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:43 smithi121 ceph-mon[88653]: Upgrade: osd.4 is safe to restart 2024-04-23T01:05:45.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:44 smithi153 ceph-mon[65847]: pgmap v142: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:45.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:45.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T01:05:45.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:45.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:44 smithi121 ceph-mon[88653]: pgmap v142: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:45.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:45.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-23T01:05:45.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:05:45.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:45 smithi153 ceph-mon[65847]: Upgrade: Updating osd.4 2024-04-23T01:05:45.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:45 smithi153 ceph-mon[65847]: Deploying daemon osd.4 on smithi153 2024-04-23T01:05:46.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:45 smithi121 ceph-mon[88653]: Upgrade: Updating osd.4 2024-04-23T01:05:46.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:45 smithi121 ceph-mon[88653]: Deploying daemon osd.4 on smithi153 2024-04-23T01:05:46.954 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:46 smithi153 ceph-mon[65847]: pgmap v143: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:05:47.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:46 smithi121 ceph-mon[88653]: pgmap v143: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:05:47.266 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:46 smithi153 systemd[1]: Stopping Ceph osd.4 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:05:47.266 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T01:05:47.115+0000 7f80ab61e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:05:47.266 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T01:05:47.115+0000 7f80ab61e700 -1 osd.4 96 *** Got signal Terminated *** 2024-04-23T01:05:47.266 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4[42805]: debug 2024-04-23T01:05:47.115+0000 7f80ab61e700 -1 osd.4 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:05:47.910 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:47 smithi153 bash[71210]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.4 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 failed (root=default,host=smithi153) (connection refused reported by osd.7) 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:47.910 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.0 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.1 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.2 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.3 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.0 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.1 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.2 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.3 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:47.911 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:47 smithi153 ceph-mon[65847]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:48.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 failed (root=default,host=smithi153) (connection refused reported by osd.7) 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.0 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.1 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.2 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.3 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.0 2024-04-23T01:05:48.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.1 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.2 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.3 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.6 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.7 2024-04-23T01:05:48.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:47 smithi121 ceph-mon[88653]: osd.4 reported immediately failed by osd.5 2024-04-23T01:05:49.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:48 smithi153 ceph-mon[65847]: pgmap v144: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:49.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:48 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:05:49.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:48 smithi153 ceph-mon[65847]: osdmap e97: 8 total, 7 up, 8 in 2024-04-23T01:05:49.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:48 smithi121 ceph-mon[88653]: pgmap v144: 129 pgs: 129 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:05:49.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:48 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:05:49.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:48 smithi121 ceph-mon[88653]: osdmap e97: 8 total, 7 up, 8 in 2024-04-23T01:05:49.794 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:49 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.4.service: Succeeded. 2024-04-23T01:05:49.794 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:49 smithi153 systemd[1]: Stopped Ceph osd.4 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:05:49.795 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:49 smithi153 systemd[1]: Starting Ceph osd.4 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:05:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:49 smithi153 ceph-mon[65847]: osdmap e98: 8 total, 7 up, 8 in 2024-04-23T01:05:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:50.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:50.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:49 smithi121 ceph-mon[88653]: osdmap e98: 8 total, 7 up, 8 in 2024-04-23T01:05:50.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:50.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:05:50.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:50 smithi153 ceph-mon[65847]: pgmap v147: 129 pgs: 4 active+undersized+degraded, 16 stale+active+clean, 5 active+undersized, 104 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 14/438 objects degraded (3.196%) 2024-04-23T01:05:50.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:50 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 14/438 objects degraded (3.196%), 4 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:50.830 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-23T01:05:50.831 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:50 smithi153 bash[71491]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-23T01:05:51.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:50 smithi121 ceph-mon[88653]: pgmap v147: 129 pgs: 4 active+undersized+degraded, 16 stale+active+clean, 5 active+undersized, 104 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 14/438 objects degraded (3.196%) 2024-04-23T01:05:51.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:50 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 14/438 objects degraded (3.196%), 4 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:52.080 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:51 smithi153 ceph-osd[71782]: -- 172.21.15.153:0/1280165465 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55848d0071e0 con 0x55848cfd6c00 2024-04-23T01:05:52.080 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:51 smithi153 bash[71744]: 8c2d0e2f414e8f9114b33447ca68309d5fd971444cf1f3ea688b00fdb0f754fe 2024-04-23T01:05:52.080 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:51 smithi153 systemd[1]: Started Ceph osd.4 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:05:52.570 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:05:52.305+0000 7f77316a17c0 -1 Falling back to public interface 2024-04-23T01:05:52.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:52 smithi153 ceph-mon[65847]: pgmap v148: 129 pgs: 15 active+undersized+degraded, 2 stale+active+clean, 38 active+undersized, 74 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 52/438 objects degraded (11.872%) 2024-04-23T01:05:52.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:52 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:52.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:52 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:52.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:52 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:52.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:52 smithi121 ceph-mon[88653]: pgmap v148: 129 pgs: 15 active+undersized+degraded, 2 stale+active+clean, 38 active+undersized, 74 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 52/438 objects degraded (11.872%) 2024-04-23T01:05:52.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:52 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:52.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:52 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:52.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:52 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:05:55.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:54 smithi153 ceph-mon[65847]: pgmap v149: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53/438 objects degraded (12.100%) 2024-04-23T01:05:55.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:54 smithi121 ceph-mon[88653]: pgmap v149: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53/438 objects degraded (12.100%) 2024-04-23T01:05:57.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: pgmap v150: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:05:57.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 53/438 objects degraded (12.100%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:57.063 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.064 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: pgmap v150: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:05:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 53/438 objects degraded (12.100%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:05:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:05:59.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:05:58 smithi153 ceph-mon[65847]: pgmap v151: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:05:59.081 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:05:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:05:59.008+0000 7f77316a17c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-23T01:05:59.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:05:58 smithi121 ceph-mon[88653]: pgmap v151: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:00.174 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:06:00.084+0000 7f77316a17c0 -1 osd.4 96 log_to_monitors true 2024-04-23T01:06:01.080 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:06:00.822+0000 7f7722afb700 -1 osd.4 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:06:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: pgmap v152: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T01:06:01.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: pgmap v152: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:01.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:01.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:01.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:02.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: pgmap v153: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 186 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: pgmap v154: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 227 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: osdmap e99: 8 total, 7 up, 8 in 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: from='osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:02.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:01 smithi153 ceph-mon[65847]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:02.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:06:02.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: pgmap v153: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 186 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:02.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: pgmap v154: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 227 B/s rd, 0 op/s; 53/438 objects degraded (12.100%) 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: osdmap e99: 8 total, 7 up, 8 in 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: from='osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:02.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:01 smithi121 ceph-mon[88653]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:03.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:02 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:06:03.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:02 smithi121 ceph-mon[88653]: osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942] boot 2024-04-23T01:06:03.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:02 smithi121 ceph-mon[88653]: osdmap e100: 8 total, 8 up, 8 in 2024-04-23T01:06:03.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:06:03.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:02 smithi121 ceph-mon[88653]: osdmap e101: 8 total, 8 up, 8 in 2024-04-23T01:06:03.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:02 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:06:03.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:02 smithi153 ceph-mon[65847]: osd.4 [v2:172.21.15.153:6800/3911448942,v1:172.21.15.153:6801/3911448942] boot 2024-04-23T01:06:03.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:02 smithi153 ceph-mon[65847]: osdmap e100: 8 total, 8 up, 8 in 2024-04-23T01:06:03.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-23T01:06:03.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:02 smithi153 ceph-mon[65847]: osdmap e101: 8 total, 8 up, 8 in 2024-04-23T01:06:04.091 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:03 smithi121 ceph-mon[88653]: pgmap v157: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 53/438 objects degraded (12.100%) 2024-04-23T01:06:04.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:03 smithi153 ceph-mon[65847]: pgmap v157: 129 pgs: 16 active+undersized+degraded, 40 active+undersized, 73 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 53/438 objects degraded (12.100%) 2024-04-23T01:06:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:05 smithi153 ceph-mon[65847]: pgmap v159: 129 pgs: 13 active+undersized+degraded, 36 active+undersized, 80 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 41/438 objects degraded (9.361%) 2024-04-23T01:06:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:06.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:05 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 41/438 objects degraded (9.361%), 13 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:06.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:05 smithi121 ceph-mon[88653]: pgmap v159: 129 pgs: 13 active+undersized+degraded, 36 active+undersized, 80 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 41/438 objects degraded (9.361%) 2024-04-23T01:06:06.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:06.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:06.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:05 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 41/438 objects degraded (9.361%), 13 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:07.588 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:06:08.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:07 smithi153 ceph-mon[65847]: pgmap v160: 129 pgs: 1 active+undersized+degraded, 128 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 1/438 objects degraded (0.228%) 2024-04-23T01:06:08.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:07 smithi121 ceph-mon[88653]: pgmap v160: 129 pgs: 1 active+undersized+degraded, 128 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 1/438 objects degraded (0.228%) 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (13m) 45s ago 24m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (3m) 45s ago 24m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (3m) 12s ago 23m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (11m) 45s ago 23m 34.3M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (18m) 45s ago 18m 16.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (18m) 12s ago 18m 42.3M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (4m) 45s ago 25m 499M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:06:08.218 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (4m) 12s ago 23m 449M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (4m) 45s ago 25m 55.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (3m) 12s ago 23m 49.5M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (15m) 45s ago 15m 192M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (15m) 12s ago 15m 85.2M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (9m) 45s ago 23m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (5m) 12s ago 22m 2545k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (2m) 45s ago 22m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (2m) 45s ago 22m 192M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (89s) 45s ago 21m 176M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (52s) 45s ago 21m 13.6M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (16s) 12s ago 21m 13.8M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (20m) 12s ago 20m 592M 4096M 16.2.0 24ecd6d5f14c f35197644ce2 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (20m) 12s ago 20m 507M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:06:08.219 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (20m) 12s ago 20m 547M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:06:08.220 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 45s ago 23m - - 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3, 2024-04-23T01:06:08.596 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 5 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 5, 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 9 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:06:08.597 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "11/23 daemons upgraded", 2024-04-23T01:06:08.920 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:06:08.921 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:06:08.921 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:06:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:08 smithi153 ceph-mon[65847]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:08 smithi153 ceph-mon[65847]: from='client.34328 172.21.15.121:0/2636903218' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:09.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:08 smithi121 ceph-mon[88653]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:08 smithi121 ceph-mon[88653]: from='client.34328 172.21.15.121:0/2636903218' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:09.302 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:06:09.303 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:06:09.303 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:06:10.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:09 smithi153 ceph-mon[65847]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:10.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:09 smithi153 ceph-mon[65847]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:10.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:09 smithi153 ceph-mon[65847]: pgmap v161: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 275 B/s rd, 0 op/s 2024-04-23T01:06:10.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:09 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/438 objects degraded (0.228%), 1 pg degraded) 2024-04-23T01:06:10.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:09 smithi153 ceph-mon[65847]: from='client.34336 172.21.15.121:0/686619628' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:06:10.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:09 smithi121 ceph-mon[88653]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:10.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:09 smithi121 ceph-mon[88653]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:10.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:09 smithi121 ceph-mon[88653]: pgmap v161: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 275 B/s rd, 0 op/s 2024-04-23T01:06:10.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:09 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/438 objects degraded (0.228%), 1 pg degraded) 2024-04-23T01:06:10.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:09 smithi121 ceph-mon[88653]: from='client.34336 172.21.15.121:0/686619628' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:06:11.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:10 smithi153 ceph-mon[65847]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:11.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:10 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:11.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:10 smithi121 ceph-mon[88653]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:11.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:10 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:12.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:11 smithi153 ceph-mon[65847]: pgmap v162: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 242 B/s rd, 0 op/s 2024-04-23T01:06:12.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:11 smithi121 ceph-mon[88653]: pgmap v162: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 242 B/s rd, 0 op/s 2024-04-23T01:06:14.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:13 smithi153 ceph-mon[65847]: pgmap v163: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:13 smithi121 ceph-mon[88653]: pgmap v163: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:16.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:15 smithi153 ceph-mon[65847]: pgmap v164: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-23T01:06:16.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:15 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:16.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:15 smithi121 ceph-mon[88653]: pgmap v164: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-23T01:06:16.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:15 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:16 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:16 smithi153 ceph-mon[65847]: Upgrade: osd.5 is safe to restart 2024-04-23T01:06:17.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:16 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-23T01:06:17.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:16 smithi121 ceph-mon[88653]: Upgrade: osd.5 is safe to restart 2024-04-23T01:06:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:17 smithi153 ceph-mon[65847]: pgmap v165: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:06:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T01:06:17.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:17 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:17 smithi121 ceph-mon[88653]: pgmap v165: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:06:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-23T01:06:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:17 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:19.048 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:18 smithi153 ceph-mon[65847]: Upgrade: Updating osd.5 2024-04-23T01:06:19.049 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:18 smithi153 ceph-mon[65847]: Deploying daemon osd.5 on smithi153 2024-04-23T01:06:19.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:18 smithi121 ceph-mon[88653]: Upgrade: Updating osd.5 2024-04-23T01:06:19.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:18 smithi121 ceph-mon[88653]: Deploying daemon osd.5 on smithi153 2024-04-23T01:06:20.035 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:20 smithi153 systemd[1]: Stopping Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:06:20.036 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:19 smithi153 ceph-mon[65847]: pgmap v166: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:20.036 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:19 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:20.036 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:19 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:20.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:19 smithi121 ceph-mon[88653]: pgmap v166: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:20.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:19 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:20.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:19 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:20.330 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T01:06:20.206+0000 7f3458ffa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:06:20.330 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T01:06:20.206+0000 7f3458ffa700 -1 osd.5 101 *** Got signal Terminated *** 2024-04-23T01:06:20.330 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5[45673]: debug 2024-04-23T01:06:20.206+0000 7f3458ffa700 -1 osd.5 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:06:21.080 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:20 smithi153 bash[73919]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.5 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.3 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 failed (root=default,host=smithi153) (connection refused reported by osd.3) 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.2 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.3 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.7 2024-04-23T01:06:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.0 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.2 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.4 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.7 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.0 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.4 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:20 smithi153 ceph-mon[65847]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.3 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 failed (root=default,host=smithi153) (connection refused reported by osd.3) 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.2 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.3 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.7 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.0 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.2 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.4 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.7 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.0 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.4 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.1 2024-04-23T01:06:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:20 smithi121 ceph-mon[88653]: osd.5 reported immediately failed by osd.6 2024-04-23T01:06:21.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:21 smithi153 ceph-mon[65847]: pgmap v167: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:21.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:21 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:06:21.817 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:21 smithi153 ceph-mon[65847]: osdmap e102: 8 total, 7 up, 8 in 2024-04-23T01:06:22.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:21 smithi121 ceph-mon[88653]: pgmap v167: 129 pgs: 129 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:22.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:21 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:06:22.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:21 smithi121 ceph-mon[88653]: osdmap e102: 8 total, 7 up, 8 in 2024-04-23T01:06:23.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:22 smithi153 ceph-mon[65847]: osdmap e103: 8 total, 7 up, 8 in 2024-04-23T01:06:23.081 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:22 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.5.service: Succeeded. 2024-04-23T01:06:23.081 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:22 smithi153 systemd[1]: Stopped Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:06:23.081 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:22 smithi153 systemd[1]: Starting Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:06:23.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:22 smithi121 ceph-mon[88653]: osdmap e103: 8 total, 7 up, 8 in 2024-04-23T01:06:24.038 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:23 smithi153 ceph-mon[65847]: pgmap v170: 129 pgs: 11 peering, 12 stale+active+clean, 106 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:06:24.039 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:23 smithi153 ceph-mon[65847]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-23T01:06:24.039 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:23 smithi153 bash[74197]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-23T01:06:24.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:23 smithi121 ceph-mon[88653]: pgmap v170: 129 pgs: 11 peering, 12 stale+active+clean, 106 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:06:24.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:23 smithi121 ceph-mon[88653]: Health check failed: Reduced data availability: 1 pg inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-23T01:06:25.080 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:24 smithi153 ceph-osd[74507]: -- 172.21.15.153:0/1827682632 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b454e6d1e0 con 0x55b454e3cc00 2024-04-23T01:06:25.080 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:24 smithi153 bash[74470]: 5ab677d63a56da831bd1668c98a0b96384e5eaa02c48f0629ec14a609cd93b7a 2024-04-23T01:06:25.080 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:24 smithi153 systemd[1]: Started Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:06:25.695 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:25 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:06:25.330+0000 7fe6e075f7c0 -1 Falling back to public interface 2024-04-23T01:06:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:25 smithi153 ceph-mon[65847]: pgmap v171: 129 pgs: 11 peering, 11 stale+active+clean, 5 active+undersized, 102 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-04-23T01:06:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:26.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:25 smithi121 ceph-mon[88653]: pgmap v171: 129 pgs: 11 peering, 11 stale+active+clean, 5 active+undersized, 102 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-04-23T01:06:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:26 smithi121 ceph-mon[88653]: pgmap v172: 129 pgs: 12 active+undersized+degraded, 5 peering, 28 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:06:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:26 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 41/438 objects degraded (9.361%), 12 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:27.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:26 smithi153 ceph-mon[65847]: pgmap v172: 129 pgs: 12 active+undersized+degraded, 5 peering, 28 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:06:27.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:26 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 41/438 objects degraded (9.361%), 12 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:28.619 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:28 smithi153 ceph-mon[65847]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-23T01:06:28.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:28 smithi121 ceph-mon[88653]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-23T01:06:29.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:29 smithi121 ceph-mon[88653]: pgmap v173: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:29.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:29 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:29.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:29 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:29.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:29 smithi153 ceph-mon[65847]: pgmap v173: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:29.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:29 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:29.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:29 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:31.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:31.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:30 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:31.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:31.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:30 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:32.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:31 smithi153 ceph-mon[65847]: pgmap v174: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:32.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:31 smithi121 ceph-mon[88653]: pgmap v174: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 216 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:32.581 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:06:32.162+0000 7fe6e075f7c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-23T01:06:33.580 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:06:33.229+0000 7fe6e075f7c0 -1 osd.5 101 log_to_monitors true 2024-04-23T01:06:34.081 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:06:33.812+0000 7fe6d1bb9700 -1 osd.5 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: pgmap v175: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 196 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:34.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:06:34.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:34.084 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.085 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: pgmap v175: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 196 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:06:34.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:34.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:06:35.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: pgmap v176: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 184 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: pgmap v177: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 224 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: osdmap e104: 8 total, 7 up, 8 in 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:35.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: pgmap v176: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 184 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: pgmap v177: 129 pgs: 14 active+undersized+degraded, 31 active+undersized, 84 active+clean; 218 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 224 B/s rd, 0 op/s; 49/438 objects degraded (11.187%) 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-23T01:06:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: osdmap e104: 8 total, 7 up, 8 in 2024-04-23T01:06:35.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:35.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:06:35.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:35.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:35.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:35 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:06:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:35 smithi153 ceph-mon[65847]: osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164] boot 2024-04-23T01:06:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:35 smithi153 ceph-mon[65847]: osdmap e105: 8 total, 8 up, 8 in 2024-04-23T01:06:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:06:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:35 smithi153 ceph-mon[65847]: osdmap e106: 8 total, 8 up, 8 in 2024-04-23T01:06:36.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:35 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:06:36.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:35 smithi121 ceph-mon[88653]: osd.5 [v2:172.21.15.153:6808/1001958164,v1:172.21.15.153:6809/1001958164] boot 2024-04-23T01:06:36.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:35 smithi121 ceph-mon[88653]: osdmap e105: 8 total, 8 up, 8 in 2024-04-23T01:06:36.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-23T01:06:36.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:35 smithi121 ceph-mon[88653]: osdmap e106: 8 total, 8 up, 8 in 2024-04-23T01:06:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:36 smithi153 ceph-mon[65847]: pgmap v180: 129 pgs: 9 peering, 11 active+undersized+degraded, 25 active+undersized, 84 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 399 B/s rd, 0 op/s; 40/438 objects degraded (9.132%) 2024-04-23T01:06:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:36 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 40/438 objects degraded (9.132%), 11 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:37.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:36 smithi121 ceph-mon[88653]: pgmap v180: 129 pgs: 9 peering, 11 active+undersized+degraded, 25 active+undersized, 84 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 399 B/s rd, 0 op/s; 40/438 objects degraded (9.132%) 2024-04-23T01:06:37.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:36 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 40/438 objects degraded (9.132%), 11 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:39.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:38 smithi153 ceph-mon[65847]: pgmap v182: 129 pgs: 18 peering, 5 active+undersized+degraded, 16 active+undersized, 90 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 19/438 objects degraded (4.338%) 2024-04-23T01:06:39.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:38 smithi121 ceph-mon[88653]: pgmap v182: 129 pgs: 18 peering, 5 active+undersized+degraded, 16 active+undersized, 90 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 19/438 objects degraded (4.338%) 2024-04-23T01:06:39.654 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:06:40.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:39 smithi121 ceph-mon[88653]: pgmap v183: 129 pgs: 18 peering, 111 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:06:40.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:39 smithi121 ceph-mon[88653]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:40.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:39 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/438 objects degraded (4.338%), 5 pgs degraded) 2024-04-23T01:06:40.287 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:06:40.287 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (14m) 77s ago 24m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:06:40.287 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (3m) 77s ago 24m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (3m) 11s ago 23m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (11m) 77s ago 24m 34.3M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (19m) 77s ago 19m 16.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (19m) 11s ago 19m 42.3M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (5m) 77s ago 26m 499M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (4m) 11s ago 23m 449M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (4m) 77s ago 26m 55.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (4m) 11s ago 23m 50.2M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (16m) 77s ago 16m 192M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (16m) 11s ago 16m 85.2M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (9m) 77s ago 24m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (5m) 11s ago 23m 6656k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:06:40.288 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (3m) 77s ago 23m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (2m) 77s ago 22m 192M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (2m) 77s ago 22m 176M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (84s) 77s ago 21m 13.6M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (48s) 11s ago 21m 211M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (15s) 11s ago 21m 11.1M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (21m) 11s ago 21m 506M 4096M 16.2.0 24ecd6d5f14c a22a56af38b0 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (20m) 11s ago 20m 547M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:06:40.289 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 77s ago 23m - - 2024-04-23T01:06:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:39 smithi153 ceph-mon[65847]: pgmap v183: 129 pgs: 18 peering, 111 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:06:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:39 smithi153 ceph-mon[65847]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:40.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:39 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 19/438 objects degraded (4.338%), 5 pgs degraded) 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:06:40.670 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2, 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 6 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 4, 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 10 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:06:40.671 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:06:40.998 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "12/23 daemons upgraded", 2024-04-23T01:06:40.999 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:06:40.999 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:06:40.999 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:06:41.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:40 smithi121 ceph-mon[88653]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:41.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:40 smithi121 ceph-mon[88653]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:41.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:40 smithi121 ceph-mon[88653]: from='client.34354 172.21.15.121:0/2076756542' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:41.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:40 smithi153 ceph-mon[65847]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:41.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:40 smithi153 ceph-mon[65847]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:41.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:40 smithi153 ceph-mon[65847]: from='client.34354 172.21.15.121:0/2076756542' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:06:41.376 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:06:41.377 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:06:41.377 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:06:42.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:41 smithi121 ceph-mon[88653]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:42.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:41 smithi121 ceph-mon[88653]: from='client.34362 172.21.15.121:0/1643066386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:06:42.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:41 smithi121 ceph-mon[88653]: pgmap v184: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 269 B/s rd, 0 op/s 2024-04-23T01:06:42.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:41 smithi153 ceph-mon[65847]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:06:42.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:41 smithi153 ceph-mon[65847]: from='client.34362 172.21.15.121:0/1643066386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:06:42.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:41 smithi153 ceph-mon[65847]: pgmap v184: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 269 B/s rd, 0 op/s 2024-04-23T01:06:44.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:44 smithi153 ceph-mon[65847]: pgmap v185: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 238 B/s rd, 0 op/s 2024-04-23T01:06:44.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:44 smithi121 ceph-mon[88653]: pgmap v185: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 238 B/s rd, 0 op/s 2024-04-23T01:06:46.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:45 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:46.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:45 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:47.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:46 smithi153 ceph-mon[65847]: pgmap v186: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:47.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:46 smithi121 ceph-mon[88653]: pgmap v186: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:49.060 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:48 smithi153 ceph-mon[65847]: pgmap v187: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 176 B/s rd, 0 op/s 2024-04-23T01:06:49.061 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:49.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:48 smithi121 ceph-mon[88653]: pgmap v187: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 176 B/s rd, 0 op/s 2024-04-23T01:06:49.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:50.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:49 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:50.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:49 smithi153 ceph-mon[65847]: Upgrade: osd.6 is safe to restart 2024-04-23T01:06:50.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:50.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:50.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:49 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-23T01:06:50.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:49 smithi121 ceph-mon[88653]: Upgrade: osd.6 is safe to restart 2024-04-23T01:06:50.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:50.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:06:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:50 smithi153 ceph-mon[65847]: pgmap v188: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T01:06:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:51.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:50 smithi121 ceph-mon[88653]: pgmap v188: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:51.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:51.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-23T01:06:51.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:06:52.009 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:51 smithi153 ceph-mon[65847]: Upgrade: Updating osd.6 2024-04-23T01:06:52.009 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:51 smithi153 ceph-mon[65847]: Deploying daemon osd.6 on smithi153 2024-04-23T01:06:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:51 smithi121 ceph-mon[88653]: Upgrade: Updating osd.6 2024-04-23T01:06:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:51 smithi121 ceph-mon[88653]: Deploying daemon osd.6 on smithi153 2024-04-23T01:06:53.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:52 smithi153 ceph-mon[65847]: pgmap v189: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:53.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:52 smithi121 ceph-mon[88653]: pgmap v189: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:53.707 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:53 smithi153 systemd[1]: Stopping Ceph osd.6 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:06:53.708 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T01:06:53.554+0000 7fad7ed6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:06:53.708 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T01:06:53.554+0000 7fad7ed6f700 -1 osd.6 106 *** Got signal Terminated *** 2024-04-23T01:06:53.708 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6[48541]: debug 2024-04-23T01:06:53.554+0000 7fad7ed6f700 -1 osd.6 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:06:54.580 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:54 smithi153 bash[76595]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.6 2024-04-23T01:06:55.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: pgmap v190: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:55.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 failed (root=default,host=smithi153) (connection refused reported by osd.7) 2024-04-23T01:06:55.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.137 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:54 smithi121 ceph-mon[88653]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: pgmap v190: 129 pgs: 129 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 failed (root=default,host=smithi153) (connection refused reported by osd.7) 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.7 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.4 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.5 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.3 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:55.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.0 2024-04-23T01:06:55.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.1 2024-04-23T01:06:55.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:54 smithi153 ceph-mon[65847]: osd.6 reported immediately failed by osd.2 2024-04-23T01:06:56.025 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:55 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:06:56.025 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:55 smithi153 ceph-mon[65847]: osdmap e107: 8 total, 7 up, 8 in 2024-04-23T01:06:56.025 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:55 smithi153 ceph-mon[65847]: osdmap e108: 8 total, 7 up, 8 in 2024-04-23T01:06:56.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:55 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:06:56.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:55 smithi121 ceph-mon[88653]: osdmap e107: 8 total, 7 up, 8 in 2024-04-23T01:06:56.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:55 smithi121 ceph-mon[88653]: osdmap e108: 8 total, 7 up, 8 in 2024-04-23T01:06:56.330 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:56 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.6.service: Succeeded. 2024-04-23T01:06:56.331 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:56 smithi153 systemd[1]: Stopped Ceph osd.6 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:06:56.331 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:56 smithi153 systemd[1]: Starting Ceph osd.6 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:06:57.108 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:56 smithi153 ceph-mon[65847]: pgmap v192: 129 pgs: 11 peering, 11 stale+active+clean, 107 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:57.108 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:56 smithi153 ceph-mon[65847]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-23T01:06:57.109 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:57 smithi153 bash[76913]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-23T01:06:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:56 smithi121 ceph-mon[88653]: pgmap v192: 129 pgs: 11 peering, 11 stale+active+clean, 107 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:06:57.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:56 smithi121 ceph-mon[88653]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-23T01:06:58.237 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:57 smithi153 ceph-mon[65847]: pgmap v194: 129 pgs: 5 active+undersized+degraded, 18 peering, 4 stale+active+clean, 9 active+undersized, 93 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7/438 objects degraded (1.598%) 2024-04-23T01:06:58.237 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:57 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 7/438 objects degraded (1.598%), 5 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:58.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:57 smithi121 ceph-mon[88653]: pgmap v194: 129 pgs: 5 active+undersized+degraded, 18 peering, 4 stale+active+clean, 9 active+undersized, 93 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7/438 objects degraded (1.598%) 2024-04-23T01:06:58.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:57 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 7/438 objects degraded (1.598%), 5 pgs degraded (PG_DEGRADED) 2024-04-23T01:06:58.580 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:58 smithi153 ceph-osd[77229]: -- 172.21.15.153:0/3468944110 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d613dfb1e0 con 0x55d613dcb000 2024-04-23T01:06:58.580 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:58 smithi153 bash[77192]: 89858230e179813e72d34823a67912e047dccdf5ef327d3b72e7ddb8865ac542 2024-04-23T01:06:58.580 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:58 smithi153 systemd[1]: Started Ceph osd.6 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:06:59.080 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:06:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:06:58.754+0000 7f84c0b297c0 -1 Falling back to public interface 2024-04-23T01:06:59.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:59.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:59.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:06:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:06:59.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:59.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:06:59.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:06:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:00.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:00 smithi153 ceph-mon[65847]: pgmap v195: 129 pgs: 9 active+undersized+degraded, 18 peering, 16 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17/438 objects degraded (3.881%) 2024-04-23T01:07:00.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:00 smithi121 ceph-mon[88653]: pgmap v195: 129 pgs: 9 active+undersized+degraded, 18 peering, 16 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17/438 objects degraded (3.881%) 2024-04-23T01:07:01.761 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:01 smithi153 ceph-mon[65847]: Health check update: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-23T01:07:01.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:01 smithi121 ceph-mon[88653]: Health check update: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-23T01:07:02.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:02 smithi153 ceph-mon[65847]: pgmap v196: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:02.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:02 smithi153 ceph-mon[65847]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-04-23T01:07:02.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:02.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:02 smithi121 ceph-mon[88653]: pgmap v196: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:02 smithi121 ceph-mon[88653]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-04-23T01:07:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:04.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:03 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:04.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:03 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:04.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:03 smithi121 ceph-mon[88653]: pgmap v197: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:04.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:03 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:04.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:03 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:04.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:03 smithi153 ceph-mon[65847]: pgmap v197: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:05.206 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:07:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:07:04.993+0000 7f84c0b297c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-23T01:07:06.080 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:07:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:07:05.920+0000 7f84c0b297c0 -1 osd.6 106 log_to_monitors true 2024-04-23T01:07:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:06.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:06.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:06.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:06.914 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:06 smithi153 ceph-mon[65847]: pgmap v198: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:06.915 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:06 smithi153 ceph-mon[65847]: from='osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T01:07:06.915 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:06 smithi153 ceph-mon[65847]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T01:07:06.915 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:06 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:06 smithi121 ceph-mon[88653]: pgmap v198: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:06 smithi121 ceph-mon[88653]: from='osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T01:07:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:06 smithi121 ceph-mon[88653]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-23T01:07:07.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:06 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 48/438 objects degraded (10.959%), 16 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:07.580 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:07:07.200+0000 7f84adf7b700 -1 osd.6 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: osdmap e109: 8 total, 7 up, 8 in 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:07:08.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='osd.6 ' entity='osd.6' 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:07 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: osdmap e109: 8 total, 7 up, 8 in 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='osd.6 ' entity='osd.6' 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:08.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:07 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: pgmap v200: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: pgmap v201: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 267 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500] boot 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: osdmap e110: 8 total, 8 up, 8 in 2024-04-23T01:07:09.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:08 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:07:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:07:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: pgmap v200: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 48/438 objects degraded (10.959%) 2024-04-23T01:07:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: pgmap v201: 129 pgs: 16 active+undersized+degraded, 27 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 267 B/s rd, 0 op/s; 48/438 objects degraded (10.959%) 2024-04-23T01:07:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:09.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-04-23T01:07:09.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:07:09.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: osd.6 [v2:172.21.15.153:6816/1339762500,v1:172.21.15.153:6817/1339762500] boot 2024-04-23T01:07:09.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: osdmap e110: 8 total, 8 up, 8 in 2024-04-23T01:07:09.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:08 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-23T01:07:10.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:10 smithi153 ceph-mon[65847]: pgmap v203: 129 pgs: 7 peering, 14 active+undersized+degraded, 22 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 362 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:07:10.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:10 smithi153 ceph-mon[65847]: osdmap e111: 8 total, 8 up, 8 in 2024-04-23T01:07:10.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:10 smithi121 ceph-mon[88653]: pgmap v203: 129 pgs: 7 peering, 14 active+undersized+degraded, 22 active+undersized, 86 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 362 B/s rd, 0 op/s; 41/438 objects degraded (9.361%) 2024-04-23T01:07:10.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:10 smithi121 ceph-mon[88653]: osdmap e111: 8 total, 8 up, 8 in 2024-04-23T01:07:11.724 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (14m) 109s ago 25m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (4m) 109s ago 25m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (4m) 10s ago 24m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (12m) 109s ago 24m 34.3M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (19m) 109s ago 19m 16.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (19m) 10s ago 19m 42.3M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (6m) 109s ago 26m 499M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (5m) 10s ago 24m 449M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (5m) 109s ago 26m 55.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (4m) 10s ago 24m 51.5M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:07:12.358 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (17m) 109s ago 17m 192M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (16m) 10s ago 16m 85.2M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (10m) 109s ago 24m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (6m) 10s ago 24m 8799k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (3m) 109s ago 23m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (3m) 109s ago 23m 192M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (2m) 109s ago 22m 176M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (116s) 109s ago 22m 13.6M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (80s) 10s ago 22m 212M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (47s) 10s ago 21m 203M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (14s) 10s ago 21m 13.7M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (21m) 10s ago 21m 547M 4096M 16.2.0 24ecd6d5f14c 769912d95f0e 2024-04-23T01:07:12.359 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 109s ago 24m - - 2024-04-23T01:07:12.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:12 smithi153 ceph-mon[65847]: pgmap v205: 129 pgs: 7 peering, 7 active+undersized+degraded, 6 active+undersized, 109 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17/438 objects degraded (3.881%) 2024-04-23T01:07:12.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:12 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 17/438 objects degraded (3.881%), 7 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:12.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:12 smithi153 ceph-mon[65847]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:12.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:12 smithi121 ceph-mon[88653]: pgmap v205: 129 pgs: 7 peering, 7 active+undersized+degraded, 6 active+undersized, 109 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17/438 objects degraded (3.881%) 2024-04-23T01:07:12.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:12 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 17/438 objects degraded (3.881%), 7 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:12.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:12 smithi121 ceph-mon[88653]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:12.737 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 7 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:07:12.738 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3, 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 11 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:07:12.739 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:07:13.061 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:07:13.062 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:07:13.062 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "13/23 daemons upgraded", 2024-04-23T01:07:13.062 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading osd daemons", 2024-04-23T01:07:13.062 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:07:13.062 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:07:13.445 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:07:13.446 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:07:13.446 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:07:13.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:13 smithi153 ceph-mon[65847]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:13.594 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:13 smithi153 ceph-mon[65847]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:13.594 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:13 smithi153 ceph-mon[65847]: from='client.34382 172.21.15.121:0/1363740981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:13.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:13 smithi121 ceph-mon[88653]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:13.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:13 smithi121 ceph-mon[88653]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:13.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:13 smithi121 ceph-mon[88653]: from='client.34382 172.21.15.121:0/1363740981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:14.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:14 smithi153 ceph-mon[65847]: pgmap v206: 129 pgs: 7 peering, 122 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-23T01:07:14.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:14 smithi153 ceph-mon[65847]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:14.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:14 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/438 objects degraded (3.881%), 7 pgs degraded) 2024-04-23T01:07:14.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:14 smithi153 ceph-mon[65847]: from='client.34390 172.21.15.121:0/4063017887' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:07:14.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:14 smithi121 ceph-mon[88653]: pgmap v206: 129 pgs: 7 peering, 122 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-04-23T01:07:14.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:14 smithi121 ceph-mon[88653]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:14.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:14 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/438 objects degraded (3.881%), 7 pgs degraded) 2024-04-23T01:07:14.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:14 smithi121 ceph-mon[88653]: from='client.34390 172.21.15.121:0/4063017887' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:07:16.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:16 smithi153 ceph-mon[65847]: pgmap v207: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:07:16.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:16 smithi121 ceph-mon[88653]: pgmap v207: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:07:18.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:18 smithi153 ceph-mon[65847]: pgmap v208: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s 2024-04-23T01:07:18.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:18 smithi121 ceph-mon[88653]: pgmap v208: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s 2024-04-23T01:07:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:20 smithi153 ceph-mon[65847]: pgmap v209: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:07:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:20 smithi121 ceph-mon[88653]: pgmap v209: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:07:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:23.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:22 smithi153 ceph-mon[65847]: pgmap v210: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 0 op/s 2024-04-23T01:07:23.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:22 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:22 smithi121 ceph-mon[88653]: pgmap v210: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 0 op/s 2024-04-23T01:07:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:22 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:24.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:23 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:24.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:23 smithi153 ceph-mon[65847]: Upgrade: osd.7 is safe to restart 2024-04-23T01:07:24.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:23 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-23T01:07:24.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:23 smithi121 ceph-mon[88653]: Upgrade: osd.7 is safe to restart 2024-04-23T01:07:24.988 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:24 smithi153 ceph-mon[65847]: pgmap v211: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:07:24.988 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:24.988 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T01:07:24.988 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:24 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:25.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:24 smithi121 ceph-mon[88653]: pgmap v211: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:07:25.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:25.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-23T01:07:25.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:24 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:26.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:25 smithi153 ceph-mon[65847]: Upgrade: Updating osd.7 2024-04-23T01:07:26.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:25 smithi153 ceph-mon[65847]: Deploying daemon osd.7 on smithi153 2024-04-23T01:07:26.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:25 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:25 smithi121 ceph-mon[88653]: Upgrade: Updating osd.7 2024-04-23T01:07:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:25 smithi121 ceph-mon[88653]: Deploying daemon osd.7 on smithi153 2024-04-23T01:07:26.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:25 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:27.067 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:26 smithi153 ceph-mon[65847]: pgmap v212: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:07:27.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:26 smithi121 ceph-mon[88653]: pgmap v212: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:07:27.331 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:27 smithi153 systemd[1]: Stopping Ceph osd.7 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:07:27.331 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T01:07:27.262+0000 7efd4bf0c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-23T01:07:27.331 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T01:07:27.262+0000 7efd4bf0c700 -1 osd.7 111 *** Got signal Terminated *** 2024-04-23T01:07:27.331 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7[51417]: debug 2024-04-23T01:07:27.262+0000 7efd4bf0c700 -1 osd.7 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:07:28.080 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:27 smithi153 bash[79315]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd.7 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.5 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 failed (root=default,host=smithi153) (connection refused reported by osd.5) 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.6 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.4 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.5 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.6 2024-04-23T01:07:28.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:27 smithi153 ceph-mon[65847]: osd.7 reported immediately failed by osd.4 2024-04-23T01:07:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.5 2024-04-23T01:07:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 failed (root=default,host=smithi153) (connection refused reported by osd.5) 2024-04-23T01:07:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.6 2024-04-23T01:07:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.4 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.5 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.0 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.1 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.2 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.3 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.6 2024-04-23T01:07:28.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:27 smithi121 ceph-mon[88653]: osd.7 reported immediately failed by osd.4 2024-04-23T01:07:29.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:28 smithi153 ceph-mon[65847]: pgmap v213: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:07:29.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:28 smithi153 ceph-mon[65847]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:07:29.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:28 smithi153 ceph-mon[65847]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2024-04-23T01:07:29.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:28 smithi153 ceph-mon[65847]: osdmap e112: 8 total, 7 up, 8 in 2024-04-23T01:07:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:28 smithi121 ceph-mon[88653]: pgmap v213: 129 pgs: 129 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:07:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:28 smithi121 ceph-mon[88653]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-23T01:07:29.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:28 smithi121 ceph-mon[88653]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2024-04-23T01:07:29.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:28 smithi121 ceph-mon[88653]: osdmap e112: 8 total, 7 up, 8 in 2024-04-23T01:07:30.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:29 smithi153 ceph-mon[65847]: osdmap e113: 8 total, 7 up, 8 in 2024-04-23T01:07:30.081 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:29 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.7.service: Succeeded. 2024-04-23T01:07:30.081 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:29 smithi153 systemd[1]: Stopped Ceph osd.7 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:07:30.081 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:29 smithi153 systemd[1]: Starting Ceph osd.7 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:07:30.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:29 smithi121 ceph-mon[88653]: osdmap e113: 8 total, 7 up, 8 in 2024-04-23T01:07:31.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:30 smithi153 ceph-mon[65847]: pgmap v216: 129 pgs: 4 active+undersized+degraded, 9 peering, 12 stale+active+clean, 6 active+undersized, 98 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15/438 objects degraded (3.425%) 2024-04-23T01:07:31.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:30 smithi153 ceph-mon[65847]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-23T01:07:31.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:30 smithi153 ceph-mon[65847]: Health check failed: Degraded data redundancy: 15/438 objects degraded (3.425%), 4 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:31.081 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-23T01:07:31.081 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-23T01:07:31.082 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-23T01:07:31.082 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-23T01:07:31.082 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-23T01:07:31.082 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-23T01:07:31.082 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:30 smithi153 bash[79598]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-23T01:07:31.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:30 smithi121 ceph-mon[88653]: pgmap v216: 129 pgs: 4 active+undersized+degraded, 9 peering, 12 stale+active+clean, 6 active+undersized, 98 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15/438 objects degraded (3.425%) 2024-04-23T01:07:31.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:30 smithi121 ceph-mon[88653]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-23T01:07:31.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:30 smithi121 ceph-mon[88653]: Health check failed: Degraded data redundancy: 15/438 objects degraded (3.425%), 4 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:32.193 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-osd[79916]: -- 172.21.15.153:0/1277140799 <== mon.1 v2:172.21.15.153:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5619e5f331e0 con 0x5619e5f03000 2024-04-23T01:07:32.580 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:32 smithi153 bash[79878]: a0b29f29af7108a9e44aa3016976df7cef124fefe0893e46219df2062d787081 2024-04-23T01:07:32.581 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:32 smithi153 systemd[1]: Started Ceph osd.7 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:07:32.971 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:07:32.629+0000 7fac191557c0 -1 Falling back to public interface 2024-04-23T01:07:32.971 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-mon[65847]: pgmap v217: 129 pgs: 14 active+undersized+degraded, 9 peering, 3 stale+active+clean, 22 active+undersized, 81 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:07:32.971 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:32.971 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:32.971 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:32 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:33.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:32 smithi121 ceph-mon[88653]: pgmap v217: 129 pgs: 14 active+undersized+degraded, 9 peering, 3 stale+active+clean, 22 active+undersized, 81 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 45/438 objects degraded (10.274%) 2024-04-23T01:07:33.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:33.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:33.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:32 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:34 smithi121 ceph-mon[88653]: pgmap v218: 129 pgs: 18 active+undersized+degraded, 9 peering, 29 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 62/438 objects degraded (14.155%) 2024-04-23T01:07:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:35.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:35.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:34 smithi153 ceph-mon[65847]: pgmap v218: 129 pgs: 18 active+undersized+degraded, 9 peering, 29 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 62/438 objects degraded (14.155%) 2024-04-23T01:07:35.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:35.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:35 smithi153 ceph-mon[65847]: pgmap v219: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:35 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 79/438 objects degraded (18.037%), 22 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:36.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:35 smithi153 ceph-mon[65847]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-23T01:07:36.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:35 smithi121 ceph-mon[88653]: pgmap v219: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:36.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:35 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 79/438 objects degraded (18.037%), 22 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:36.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:35 smithi121 ceph-mon[88653]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-23T01:07:37.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.984 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.985 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.985 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:37.985 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:38.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:38 smithi153 ceph-mon[65847]: pgmap v220: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:39.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:38 smithi121 ceph-mon[88653]: pgmap v220: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:40.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:07:40.707+0000 7fac191557c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-23T01:07:41.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:40 smithi153 ceph-mon[65847]: pgmap v221: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:41.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:40 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:41.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:40 smithi121 ceph-mon[88653]: pgmap v221: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:41.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:40 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.330 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:07:41.894+0000 7fac191557c0 -1 osd.7 111 log_to_monitors true 2024-04-23T01:07:42.331 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:07:42.105+0000 7fac0a5af700 -1 osd.7 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:07:42.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: pgmap v222: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 172 B/s rd, 0 op/s; 79/438 objects degraded (18.037%) 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: pgmap v223: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s; 79/438 objects degraded (18.037%) 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all osd 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-23T01:07:42.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: Upgrade: Setting require_osd_release to 18 reef 2024-04-23T01:07:42.333 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-04-23T01:07:42.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: pgmap v222: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 172 B/s rd, 0 op/s; 79/438 objects degraded (18.037%) 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: pgmap v223: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s; 79/438 objects degraded (18.037%) 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all osd 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-23T01:07:42.387 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-23T01:07:42.388 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-23T01:07:42.388 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-23T01:07:42.388 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-23T01:07:42.388 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: Upgrade: Setting require_osd_release to 18 reef 2024-04-23T01:07:42.388 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-04-23T01:07:43.243 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: osdmap e114: 8 total, 7 up, 8 in 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi121.vupjhr"]}]: dispatch 2024-04-23T01:07:43.244 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:42 smithi121 ceph-mon[88653]: Upgrade: It appears safe to stop mds.foofs.smithi121.vupjhr 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: osdmap e114: 8 total, 7 up, 8 in 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi153", "root=default"]}]: dispatch 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi121.vupjhr"]}]: dispatch 2024-04-23T01:07:43.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:42 smithi153 ceph-mon[65847]: Upgrade: It appears safe to stop mds.foofs.smithi121.vupjhr 2024-04-23T01:07:43.804 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:07:44.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: pgmap v225: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:44.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:07:44.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691] boot 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: osdmap e115: 8 total, 8 up, 8 in 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: Upgrade: Updating mds.foofs.smithi121.vupjhr 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: Deploying daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T01:07:44.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:43 smithi121 ceph-mon[88653]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:44.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: pgmap v225: 129 pgs: 22 active+undersized+degraded, 34 active+undersized, 73 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 79/438 objects degraded (18.037%) 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: osd.7 [v2:172.21.15.153:6824/2970367691,v1:172.21.15.153:6825/2970367691] boot 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: osdmap e115: 8 total, 8 up, 8 in 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: Upgrade: Updating mds.foofs.smithi121.vupjhr 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi121.vupjhr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: Deploying daemon mds.foofs.smithi121.vupjhr on smithi121 2024-04-23T01:07:44.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:43 smithi153 ceph-mon[65847]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:44.492 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:07:44.492 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (15m) 2m ago 25m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (4m) 2m ago 25m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (4m) 7s ago 24m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (12m) 2m ago 25m 34.3M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (20m) 2m ago 20m 16.1M - 16.2.0 24ecd6d5f14c bd3076f247be 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (20m) 7s ago 20m 42.3M - 16.2.0 24ecd6d5f14c d791b97a8654 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (6m) 2m ago 27m 499M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (6m) 7s ago 24m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (5m) 2m ago 27m 55.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (5m) 7s ago 24m 51.1M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (17m) 2m ago 17m 192M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (17m) 7s ago 17m 85.2M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (10m) 2m ago 25m 15.1M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (6m) 7s ago 24m 8799k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:07:44.493 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (4m) 2m ago 24m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (3m) 2m ago 23m 192M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (3m) 2m ago 23m 176M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (2m) 2m ago 23m 13.6M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (113s) 7s ago 22m 214M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (80s) 7s ago 22m 204M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (46s) 7s ago 22m 197M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (12s) 7s ago 21m 10.3M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:07:44.494 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 unknown 2m ago 25m - - 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:07:44.881 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2, 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 12 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:07:44.882 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:07:45.207 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:07:45.207 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "osd", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "mon" 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "14/23 daemons upgraded", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading mds daemons", 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:07:45.208 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:07:45.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:45 smithi121 ceph-mon[88653]: osdmap e116: 8 total, 8 up, 8 in 2024-04-23T01:07:45.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:45 smithi121 ceph-mon[88653]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:45.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:45 smithi121 ceph-mon[88653]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:45.223 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:45 smithi121 ceph-mon[88653]: from='client.34410 172.21.15.121:0/3269058363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:45.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:45 smithi153 ceph-mon[65847]: osdmap e116: 8 total, 8 up, 8 in 2024-04-23T01:07:45.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:45 smithi153 ceph-mon[65847]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:45.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:45 smithi153 ceph-mon[65847]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:45.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:45 smithi153 ceph-mon[65847]: from='client.34410 172.21.15.121:0/3269058363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 56/438 objects degraded (12.785%), 15 pgs degraded 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in unknown state 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 56/438 objects degraded (12.785%), 15 pgs degraded 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-23T01:07:45.628 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-23T01:07:45.629 INFO:teuthology.orchestra.run.smithi121.stdout: pg 5.19 is active+undersized+degraded, acting [1,5] 2024-04-23T01:07:46.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: pgmap v228: 129 pgs: 14 peering, 15 active+undersized+degraded, 18 active+undersized, 82 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 56/438 objects degraded (12.785%) 2024-04-23T01:07:46.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: Health check update: Degraded data redundancy: 56/438 objects degraded (12.785%), 15 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:46.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:46.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: daemon mds.foofs.smithi121.vupjhr compat changed unexpectedly 2024-04-23T01:07:46.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: osdmap e117: 8 total, 8 up, 8 in 2024-04-23T01:07:46.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:46 smithi121 ceph-mon[88653]: from='client.34418 172.21.15.121:0/3993521040' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:07:46.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: pgmap v228: 129 pgs: 14 peering, 15 active+undersized+degraded, 18 active+undersized, 82 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 56/438 objects degraded (12.785%) 2024-04-23T01:07:46.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: Health check update: Degraded data redundancy: 56/438 objects degraded (12.785%), 15 pgs degraded (PG_DEGRADED) 2024-04-23T01:07:46.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:07:46.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: daemon mds.foofs.smithi121.vupjhr compat changed unexpectedly 2024-04-23T01:07:46.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: osdmap e117: 8 total, 8 up, 8 in 2024-04-23T01:07:46.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:46 smithi153 ceph-mon[65847]: from='client.34418 172.21.15.121:0/3993521040' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:07:47.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:47 smithi121 ceph-mon[88653]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-23T01:07:47.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:47 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 2024-04-23T01:07:47.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:47 smithi153 ceph-mon[65847]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-23T01:07:47.583 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:47 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 2024-04-23T01:07:48.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:48 smithi121 ceph-mon[88653]: pgmap v230: 129 pgs: 14 peering, 7 active+undersized+degraded, 10 active+undersized, 98 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 425 B/s rd, 0 op/s; 32/438 objects degraded (7.306%) 2024-04-23T01:07:48.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:48 smithi153 ceph-mon[65847]: pgmap v230: 129 pgs: 14 peering, 7 active+undersized+degraded, 10 active+undersized, 98 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 425 B/s rd, 0 op/s; 32/438 objects degraded (7.306%) 2024-04-23T01:07:49.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:49 smithi153 ceph-mon[65847]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/438 objects degraded (7.306%), 7 pgs degraded) 2024-04-23T01:07:49.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:49 smithi121 ceph-mon[88653]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/438 objects degraded (7.306%), 7 pgs degraded) 2024-04-23T01:07:51.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: pgmap v231: 129 pgs: 7 peering, 122 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:07:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:50 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:51.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: pgmap v231: 129 pgs: 7 peering, 122 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:07:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:07:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:50 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:07:51.845 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:51 smithi121 ceph-mon[88653]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-23T01:07:51.845 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:51 smithi121 ceph-mon[88653]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:boot 2024-04-23T01:07:51.845 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:51 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:07:51.845 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:51 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:07:52.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:51 smithi153 ceph-mon[65847]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-23T01:07:52.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:51 smithi153 ceph-mon[65847]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:boot 2024-04-23T01:07:52.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:51 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi153.uunfav=up:active} 1 up:standby 2024-04-23T01:07:52.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:51 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:07:53.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:52 smithi153 ceph-mon[65847]: pgmap v232: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 262 B/s rd, 0 op/s 2024-04-23T01:07:53.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:52 smithi121 ceph-mon[88653]: pgmap v232: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 262 B/s rd, 0 op/s 2024-04-23T01:07:54.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:54 smithi121 ceph-mon[88653]: pgmap v233: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-23T01:07:55.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:54 smithi153 ceph-mon[65847]: pgmap v233: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-23T01:07:56.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:56 smithi153 ceph-mon[65847]: pgmap v234: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:07:56.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:56 smithi121 ceph-mon[88653]: pgmap v234: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:07:56.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:56.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:07:58.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:07:58 smithi153 ceph-mon[65847]: pgmap v235: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-04-23T01:07:58.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:07:58 smithi121 ceph-mon[88653]: pgmap v235: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: pgmap v236: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: Detected new or changed devices on smithi121 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: pgmap v237: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-04-23T01:08:00.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: pgmap v238: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 237 B/s rd, 0 op/s 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi153.uunfav"]}]: dispatch 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: Upgrade: It appears safe to stop mds.foofs.smithi153.uunfav 2024-04-23T01:08:00.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: pgmap v236: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: Detected new or changed devices on smithi121 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: pgmap v237: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: pgmap v238: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 237 B/s rd, 0 op/s 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:00.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi153.uunfav"]}]: dispatch 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: Upgrade: It appears safe to stop mds.foofs.smithi153.uunfav 2024-04-23T01:08:00.887 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:02.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: Upgrade: Updating mds.foofs.smithi153.uunfav 2024-04-23T01:08:02.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:02.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:08:02.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:02.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: Deploying daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T01:08:02.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:02 smithi121 ceph-mon[88653]: pgmap v239: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 237 B/s rd, 0 op/s 2024-04-23T01:08:02.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: Upgrade: Updating mds.foofs.smithi153.uunfav 2024-04-23T01:08:02.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:02.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi153.uunfav", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-23T01:08:02.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:02.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: Deploying daemon mds.foofs.smithi153.uunfav on smithi153 2024-04-23T01:08:02.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:02 smithi153 ceph-mon[65847]: pgmap v239: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 237 B/s rd, 0 op/s 2024-04-23T01:08:03.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: daemon mds.foofs.smithi153.uunfav compat changed unexpectedly 2024-04-23T01:08:03.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-23T01:08:03.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T01:08:03.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: osdmap e118: 8 total, 8 up, 8 in 2024-04-23T01:08:03.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: Standby daemon mds.foofs.smithi121.vupjhr assigned to filesystem foofs as rank 0 2024-04-23T01:08:03.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-23T01:08:03.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-23T01:08:03.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-23T01:08:03.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:03 smithi153 ceph-mon[65847]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:replay} 2024-04-23T01:08:03.885 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:08:03.452+0000 7f3c1ad1a700 -1 bdev(0x56322cbb8000 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 1 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi121[88630]: 2024-04-23T01:08:03.421+0000 7f76cc355700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: daemon mds.foofs.smithi153.uunfav compat changed unexpectedly 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: osdmap e118: 8 total, 8 up, 8 in 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: Standby daemon mds.foofs.smithi121.vupjhr assigned to filesystem foofs as rank 0 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-23T01:08:03.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:03 smithi121 ceph-mon[88653]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:replay} 2024-04-23T01:08:04.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:04 smithi153 ceph-mon[65847]: pgmap v241: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-23T01:08:04.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:04 smithi121 ceph-mon[88653]: pgmap v241: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: reconnect by client.14760 172.21.15.121:0/322479744 after 0.000999988 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: reconnect by client.24525 172.21.15.153:0/1889087175 after 0.000999988 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:reconnect 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:reconnect} 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:05.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:05 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:05.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: reconnect by client.14760 172.21.15.121:0/322479744 after 0.000999988 2024-04-23T01:08:05.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: reconnect by client.24525 172.21.15.153:0/1889087175 after 0.000999988 2024-04-23T01:08:05.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:reconnect 2024-04-23T01:08:05.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:reconnect} 2024-04-23T01:08:05.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:05.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:05 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:06.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:06 smithi153 ceph-mon[65847]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:rejoin 2024-04-23T01:08:06.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:06 smithi153 ceph-mon[65847]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:rejoin} 2024-04-23T01:08:06.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:06 smithi153 ceph-mon[65847]: daemon mds.foofs.smithi121.vupjhr is now active in filesystem foofs as rank 0 2024-04-23T01:08:06.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:06 smithi153 ceph-mon[65847]: pgmap v242: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 5 op/s 2024-04-23T01:08:06.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:06 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:06.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:06 smithi121 ceph-mon[88653]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:rejoin 2024-04-23T01:08:06.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:06 smithi121 ceph-mon[88653]: fsmap foofs:1/1 {0=foofs.smithi121.vupjhr=up:rejoin} 2024-04-23T01:08:06.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:06 smithi121 ceph-mon[88653]: daemon mds.foofs.smithi121.vupjhr is now active in filesystem foofs as rank 0 2024-04-23T01:08:06.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:06 smithi121 ceph-mon[88653]: pgmap v242: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 5 op/s 2024-04-23T01:08:06.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:06 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:07.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:07 smithi153 ceph-mon[65847]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-23T01:08:07.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:07 smithi153 ceph-mon[65847]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:active 2024-04-23T01:08:07.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:07 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi121.vupjhr=up:active} 2024-04-23T01:08:07.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:07 smithi121 ceph-mon[88653]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-23T01:08:07.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:07 smithi121 ceph-mon[88653]: mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740] up:active 2024-04-23T01:08:07.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:07 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi121.vupjhr=up:active} 2024-04-23T01:08:08.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:08 smithi153 ceph-mon[65847]: pgmap v243: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 255 B/s wr, 10 op/s 2024-04-23T01:08:08.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:08 smithi121 ceph-mon[88653]: pgmap v243: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 255 B/s wr, 10 op/s 2024-04-23T01:08:10.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:10 smithi153 ceph-mon[65847]: pgmap v244: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 8 op/s 2024-04-23T01:08:10.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:10 smithi121 ceph-mon[88653]: pgmap v244: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 8 op/s 2024-04-23T01:08:12.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:12 smithi121 ceph-mon[88653]: pgmap v245: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-04-23T01:08:13.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:12 smithi153 ceph-mon[65847]: pgmap v245: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: mds.? [v2:172.21.15.153:6832/1414183216,v1:172.21.15.153:6833/1414183216] up:boot 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: fsmap foofs:1 {0=foofs.smithi121.vupjhr=up:active} 1 up:standby 2024-04-23T01:08:14.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:13 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:14.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: mds.? [v2:172.21.15.153:6832/1414183216,v1:172.21.15.153:6833/1414183216] up:boot 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: fsmap foofs:1 {0=foofs.smithi121.vupjhr=up:active} 1 up:standby 2024-04-23T01:08:14.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:13 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:15.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:14 smithi153 ceph-mon[65847]: pgmap v246: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-04-23T01:08:15.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:14 smithi121 ceph-mon[88653]: pgmap v246: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-04-23T01:08:15.982 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (15m) 21s ago 26m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (5m) 21s ago 26m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (5m) 0s ago 25m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (13m) 21s ago 26m 34.8M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (27s) 21s ago 20m 11.0M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:08:16.618 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (4s) 0s ago 20m 10.1M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (7m) 21s ago 27m 503M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (6m) 0s ago 25m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (6m) 21s ago 27m 76.2M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (5m) 0s ago 25m 56.0M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (18m) 21s ago 18m 193M - 3.5 24ecd6d5f14c 86f552b22d78 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (18m) 0s ago 18m 85.3M - 3.5 24ecd6d5f14c 9a6fe2b35c5c 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (11m) 21s ago 25m 14.5M - 0.18.1 e5a616e4b9cf be5f08554de8 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (7m) 0s ago 25m 8799k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (4m) 21s ago 24m 216M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (4m) 21s ago 24m 197M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (3m) 21s ago 23m 179M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (3m) 21s ago 23m 211M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (2m) 0s ago 23m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (112s) 0s ago 22m 210M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:08:16.619 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (78s) 0s ago 22m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:08:16.620 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (44s) 0s ago 22m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:08:16.620 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 error 21s ago 25m - - 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:08:17.003 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:17.004 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:17.004 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:08:17.004 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:08:17.004 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:08:17.004 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:08:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:16 smithi153 ceph-mon[65847]: pgmap v247: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-04-23T01:08:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:16 smithi121 ceph-mon[88653]: pgmap v247: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-04-23T01:08:17.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "osd", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "mds", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "mon", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:08:17.325 INFO:teuthology.orchestra.run.smithi121.stdout: "crash" 2024-04-23T01:08:17.326 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:08:17.326 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "16/23 daemons upgraded", 2024-04-23T01:08:17.326 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading mds daemons", 2024-04-23T01:08:17.326 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:08:17.326 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:08:17.707 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:08:17.707 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:08:17.707 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in error state 2024-04-23T01:08:18.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:17 smithi153 ceph-mon[65847]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:17 smithi153 ceph-mon[65847]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:17 smithi153 ceph-mon[65847]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:17 smithi153 ceph-mon[65847]: from='client.34438 172.21.15.121:0/4058954176' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:18.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:17 smithi153 ceph-mon[65847]: from='client.34446 172.21.15.121:0/1530287858' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:08:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:17 smithi121 ceph-mon[88653]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:17 smithi121 ceph-mon[88653]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:17 smithi121 ceph-mon[88653]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:18.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:17 smithi121 ceph-mon[88653]: from='client.34438 172.21.15.121:0/4058954176' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:18.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:17 smithi121 ceph-mon[88653]: from='client.34446 172.21.15.121:0/1530287858' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:08:19.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:18 smithi153 ceph-mon[65847]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:19.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:18 smithi153 ceph-mon[65847]: pgmap v248: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-04-23T01:08:19.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:18 smithi121 ceph-mon[88653]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:19.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:18 smithi121 ceph-mon[88653]: pgmap v248: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: pgmap v249: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:20.961 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi121.vupjhr"}]': finished 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:20.962 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi153.uunfav"}]': finished 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:20.963 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: pgmap v249: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-23T01:08:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:21.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi121.vupjhr"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi121.vupjhr"}]': finished 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi153.uunfav"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi153.uunfav"}]': finished 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:21.083 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Detected new or changed devices on smithi153 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: pgmap v250: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 8.0 KiB/s wr, 5 op/s 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: pgmap v251: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s rd, 0 op/s 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all mds 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all rgw 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all rbd-mirror 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all ceph-exporter 2024-04-23T01:08:21.993 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:21 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all iscsi 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Detected new or changed devices on smithi153 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: pgmap v250: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 8.0 KiB/s wr, 5 op/s 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: pgmap v251: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s rd, 0 op/s 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all mds 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all rgw 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all rbd-mirror 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all ceph-exporter 2024-04-23T01:08:22.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:21 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all iscsi 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Upgrade: Updating nfs.foo.0.0.smithi121.aanwei (1/2) 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.0.0.smithi121.aanwei 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Rados config object exists: conf-nfs.foo 2024-04-23T01:08:23.319 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T01:08:23.320 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T01:08:23.320 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T01:08:23.320 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:23.320 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T01:08:23.320 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:23 smithi121 ceph-mon[88653]: pgmap v252: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s rd, 235 B/s wr, 0 op/s 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Upgrade: Updating nfs.foo.0.0.smithi121.aanwei (1/2) 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.0.0.smithi121.aanwei 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Rados config object exists: conf-nfs.foo 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.0.0.smithi121.aanwei-rgw 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi121.aanwei-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Bind address in nfs.foo.0.0.smithi121.aanwei's ganesha conf is defaulting to empty 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: Deploying daemon nfs.foo.0.0.smithi121.aanwei on smithi121 2024-04-23T01:08:23.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:23 smithi153 ceph-mon[65847]: pgmap v252: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s rd, 235 B/s wr, 0 op/s 2024-04-23T01:08:25.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:25 smithi153 ceph-mon[65847]: pgmap v253: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s wr, 0 op/s 2024-04-23T01:08:25.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:25 smithi121 ceph-mon[88653]: pgmap v253: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 235 B/s wr, 0 op/s 2024-04-23T01:08:26.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:26 smithi153 ceph-mon[65847]: Evicting (and blocklisting) client session 14760 (172.21.15.121:0/322479744) 2024-04-23T01:08:26.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:26 smithi153 ceph-mon[65847]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/322479744"}]: dispatch 2024-04-23T01:08:26.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:26 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:26.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:26 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:26.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:26 smithi121 ceph-mon[88653]: Evicting (and blocklisting) client session 14760 (172.21.15.121:0/322479744) 2024-04-23T01:08:26.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:26 smithi121 ceph-mon[88653]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/322479744"}]: dispatch 2024-04-23T01:08:26.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:26 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:26.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:26 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:27.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:27 smithi153 ceph-mon[65847]: pgmap v254: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 707 B/s wr, 1 op/s 2024-04-23T01:08:27.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:27 smithi153 ceph-mon[65847]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/322479744"}]': finished 2024-04-23T01:08:27.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:27 smithi153 ceph-mon[65847]: osdmap e119: 8 total, 8 up, 8 in 2024-04-23T01:08:27.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:27 smithi121 ceph-mon[88653]: pgmap v254: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 707 B/s wr, 1 op/s 2024-04-23T01:08:27.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:27 smithi121 ceph-mon[88653]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.121:0/322479744"}]': finished 2024-04-23T01:08:27.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:27 smithi121 ceph-mon[88653]: osdmap e119: 8 total, 8 up, 8 in 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Upgrade: Updating nfs.foo.1.0.smithi153.xxakvf (2/2) 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T01:08:28.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Rados config object exists: conf-nfs.foo 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:28.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:28 smithi153 ceph-mon[65847]: Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T01:08:28.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Upgrade: Updating nfs.foo.1.0.smithi153.xxakvf (2/2) 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Rados config object exists: conf-nfs.foo 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Creating key for client.nfs.foo.1.0.smithi153.xxakvf-rgw 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi153.xxakvf-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Bind address in nfs.foo.1.0.smithi153.xxakvf's ganesha conf is defaulting to empty 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:28.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:28 smithi121 ceph-mon[88653]: Deploying daemon nfs.foo.1.0.smithi153.xxakvf on smithi153 2024-04-23T01:08:29.754 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:29 smithi153 ceph-mon[65847]: pgmap v256: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-23T01:08:29.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:29 smithi121 ceph-mon[88653]: pgmap v256: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-23T01:08:31.747 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: pgmap v257: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: Evicting (and blocklisting) client session 24525 (172.21.15.153:0/1889087175) 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/1889087175"}]: dispatch 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:31 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:31.748 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: pgmap v257: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-23T01:08:31.749 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: Evicting (and blocklisting) client session 24525 (172.21.15.153:0/1889087175) 2024-04-23T01:08:31.749 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/1889087175"}]: dispatch 2024-04-23T01:08:31.749 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:31.749 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:31.749 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:31 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:32.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:32 smithi153 ceph-mon[65847]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/1889087175"}]': finished 2024-04-23T01:08:32.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:32 smithi153 ceph-mon[65847]: osdmap e120: 8 total, 8 up, 8 in 2024-04-23T01:08:32.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:32 smithi121 ceph-mon[88653]: from='mds.? [v2:172.21.15.121:6834/1252946740,v1:172.21.15.121:6835/1252946740]' entity='mds.foofs.smithi121.vupjhr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.153:0/1889087175"}]': finished 2024-04-23T01:08:32.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:32 smithi121 ceph-mon[88653]: osdmap e120: 8 total, 8 up, 8 in 2024-04-23T01:08:33.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:33 smithi153 ceph-mon[65847]: pgmap v259: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-04-23T01:08:33.850 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:33 smithi121 ceph-mon[88653]: pgmap v259: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: pgmap v260: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:35.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: pgmap v260: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.1 KiB/s wr, 4 op/s 2024-04-23T01:08:35.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:35.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:35.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:37 smithi153 ceph-mon[65847]: pgmap v261: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-23T01:08:37.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.582 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.585 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.585 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.585 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:37 smithi121 ceph-mon[88653]: pgmap v261: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-23T01:08:37.586 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:37.586 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:39.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:39 smithi153 ceph-mon[65847]: pgmap v262: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-23T01:08:39.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:39.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:39 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:39.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:39 smithi121 ceph-mon[88653]: pgmap v262: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-23T01:08:39.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:39.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:39 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.790 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: pgmap v263: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: pgmap v264: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.4 KiB/s rd, 680 B/s wr, 2 op/s 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: pgmap v265: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.6 KiB/s rd, 747 B/s wr, 2 op/s 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:41.791 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all nfs 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi121.aanwei"}]: dispatch 2024-04-23T01:08:41.792 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi121.aanwei"}]': finished 2024-04-23T01:08:41.793 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi153.xxakvf"}]: dispatch 2024-04-23T01:08:41.793 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi153.xxakvf"}]': finished 2024-04-23T01:08:41.793 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.793 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: Upgrade: Setting container_image for all nvmeof 2024-04-23T01:08:41.793 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:41 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: pgmap v263: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-04-23T01:08:41.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: pgmap v264: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.4 KiB/s rd, 680 B/s wr, 2 op/s 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: pgmap v265: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.6 KiB/s rd, 747 B/s wr, 2 op/s 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all nfs 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi121.aanwei"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi121.aanwei"}]': finished 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi153.xxakvf"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi153.xxakvf"}]': finished 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: Upgrade: Setting container_image for all nvmeof 2024-04-23T01:08:41.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:41 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:43.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:43 smithi121 ceph-mon[88653]: Upgrade: Updating node-exporter.smithi121 (1/2) 2024-04-23T01:08:43.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:43 smithi121 ceph-mon[88653]: Deploying daemon node-exporter.smithi121 on smithi121 2024-04-23T01:08:43.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:43 smithi121 ceph-mon[88653]: pgmap v266: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 871 B/s wr, 2 op/s 2024-04-23T01:08:43.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:43 smithi153 ceph-mon[65847]: Upgrade: Updating node-exporter.smithi121 (1/2) 2024-04-23T01:08:43.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:43 smithi153 ceph-mon[65847]: Deploying daemon node-exporter.smithi121 on smithi121 2024-04-23T01:08:43.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:43 smithi153 ceph-mon[65847]: pgmap v266: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 871 B/s wr, 2 op/s 2024-04-23T01:08:45.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:45 smithi153 ceph-mon[65847]: pgmap v267: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 498 B/s wr, 0 op/s 2024-04-23T01:08:45.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:45 smithi121 ceph-mon[88653]: pgmap v267: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 498 B/s wr, 0 op/s 2024-04-23T01:08:47.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:47 smithi153 ceph-mon[65847]: pgmap v268: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 996 B/s rd, 622 B/s wr, 1 op/s 2024-04-23T01:08:47.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:47 smithi121 ceph-mon[88653]: pgmap v268: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 996 B/s rd, 622 B/s wr, 1 op/s 2024-04-23T01:08:48.074 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (16m) 12s ago 26m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (5m) 12s ago 26m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (5m) 14s ago 25m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (14m) 12s ago 26m 34.8M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (59s) 12s ago 21m 26.3M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (36s) 14s ago 21m 11.4M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (7m) 12s ago 28m 505M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:08:48.706 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (7m) 14s ago 25m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (6m) 12s ago 28m 78.5M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (6m) 14s ago 25m 57.7M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (23s) 12s ago 18m 63.3M - 5.5 9900ed1a6b81 f8bf9272bcd6 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (18s) 14s ago 18m 26.8M - 5.5 9900ed1a6b81 0330629fef30 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 starting - - - - 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (7m) 14s ago 25m 8799k - 0.18.1 e5a616e4b9cf bacda7ab9af5 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (5m) 12s ago 25m 218M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (4m) 12s ago 24m 201M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (4m) 12s ago 24m 184M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (3m) 12s ago 24m 214M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (2m) 14s ago 23m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (2m) 14s ago 23m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (110s) 14s ago 23m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:08:48.707 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (77s) 14s ago 22m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:08:48.708 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 error 12s ago 26m - - 2024-04-23T01:08:49.060 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:49.060 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:48 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:49.085 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:08:49.086 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:08:49.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:49.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:48 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "osd", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "mds", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "mon", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:08:49.404 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout: "nfs" 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "18/23 daemons upgraded", 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:08:49.405 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:08:49.792 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-23T01:08:49.792 INFO:teuthology.orchestra.run.smithi121.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-23T01:08:49.792 INFO:teuthology.orchestra.run.smithi121.stdout: daemon prometheus.smithi121 on smithi121 is in error state 2024-04-23T01:08:50.068 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: pgmap v269: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 996 B/s rd, 622 B/s wr, 1 op/s 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='client.34504 172.21.15.121:0/4191957634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:50.069 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:49 smithi153 ceph-mon[65847]: from='client.34512 172.21.15.121:0/2489488147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: pgmap v269: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 996 B/s rd, 622 B/s wr, 1 op/s 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='client.34504 172.21.15.121:0/4191957634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:08:50.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:49 smithi121 ceph-mon[88653]: from='client.34512 172.21.15.121:0/2489488147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:08:51.020 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:50 smithi153 ceph-mon[65847]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:51.020 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:50 smithi153 ceph-mon[65847]: Upgrade: Updating node-exporter.smithi153 (2/2) 2024-04-23T01:08:51.020 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:50 smithi153 ceph-mon[65847]: Deploying daemon node-exporter.smithi153 on smithi153 2024-04-23T01:08:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:50 smithi121 ceph-mon[88653]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:08:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:50 smithi121 ceph-mon[88653]: Upgrade: Updating node-exporter.smithi153 (2/2) 2024-04-23T01:08:51.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:50 smithi121 ceph-mon[88653]: Deploying daemon node-exporter.smithi153 on smithi153 2024-04-23T01:08:52.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:51 smithi153 ceph-mon[65847]: pgmap v270: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 818 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T01:08:52.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:51 smithi121 ceph-mon[88653]: pgmap v270: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 818 B/s rd, 511 B/s wr, 1 op/s 2024-04-23T01:08:54.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:53 smithi153 ceph-mon[65847]: pgmap v271: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-04-23T01:08:54.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:53 smithi121 ceph-mon[88653]: pgmap v271: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-04-23T01:08:55.494 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:55 smithi121 ceph-mon[88653]: pgmap v272: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-04-23T01:08:55.495 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:55.495 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:55.495 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:55.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:55 smithi153 ceph-mon[65847]: pgmap v272: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-04-23T01:08:55.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:55.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:55.581 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:08:57.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:57 smithi153 ceph-mon[65847]: pgmap v273: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-04-23T01:08:57.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:57 smithi121 ceph-mon[88653]: pgmap v273: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-04-23T01:08:58.804 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:58 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.804 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:58 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.804 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:58 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.804 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:58 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:58 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:58 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:58 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:58.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:58 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:59.672 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:59 smithi153 ceph-mon[65847]: pgmap v274: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:08:59.672 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:59.672 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:08:59 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:59.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:59 smithi121 ceph-mon[88653]: pgmap v274: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:08:59.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:08:59.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:08:59 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:01.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:01.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:00 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:01.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:00 smithi153 ceph-mon[65847]: pgmap v275: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:01.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:01.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:00 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:01.386 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:00 smithi121 ceph-mon[88653]: pgmap v275: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:02.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:02 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:03.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:03.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:02 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:03.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:03 smithi121 ceph-mon[88653]: pgmap v276: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:04.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:03 smithi153 ceph-mon[65847]: pgmap v276: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:05.018 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: pgmap v277: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: pgmap v278: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.019 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.020 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: pgmap v277: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: pgmap v278: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:05.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:05.082 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:07.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:06 smithi153 ceph-mon[65847]: Upgrade: Updating prometheus.smithi121 2024-04-23T01:09:07.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:06 smithi153 ceph-mon[65847]: Deploying daemon prometheus.smithi121 on smithi121 2024-04-23T01:09:07.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:06 smithi153 ceph-mon[65847]: pgmap v279: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:06 smithi121 ceph-mon[88653]: Upgrade: Updating prometheus.smithi121 2024-04-23T01:09:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:06 smithi121 ceph-mon[88653]: Deploying daemon prometheus.smithi121 on smithi121 2024-04-23T01:09:07.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:06 smithi121 ceph-mon[88653]: pgmap v279: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:09.033 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:08 smithi121 ceph-mon[88653]: pgmap v280: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:09.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:08 smithi153 ceph-mon[65847]: pgmap v280: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:10.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:10 smithi121 ceph-mon[88653]: pgmap v281: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:10.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:10 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:10.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:10 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:10.886 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:10 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:11.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:10 smithi153 ceph-mon[65847]: pgmap v281: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:11.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:10 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:11.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:10 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:11.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:10 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:13.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:12 smithi153 ceph-mon[65847]: pgmap v282: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:13.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:12 smithi121 ceph-mon[88653]: pgmap v282: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-23T01:09:15.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:14 smithi153 ceph-mon[65847]: pgmap v283: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:09:15.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:14 smithi121 ceph-mon[88653]: pgmap v283: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-23T01:09:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:16 smithi153 ceph-mon[65847]: pgmap v284: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:17.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:16 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.081 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.081 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.082 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:16 smithi121 ceph-mon[88653]: pgmap v284: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:17.082 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:17.082 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:16 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:19.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:18 smithi153 ceph-mon[65847]: pgmap v285: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:19.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:18 smithi121 ceph-mon[88653]: pgmap v285: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:20.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:19 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:20.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:19 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:20.175 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (16m) 5s ago 27m 18.0M - 0.20.0 0881eb8f169f 6ecc90038692 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (6m) 5s ago 27m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (6m) 23s ago 26m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (14m) 5s ago 27m 34.8M - 6.7.4 557c83e11646 bf8f2a8be2bd 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (91s) 5s ago 21m 26.6M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:09:20.831 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (68s) 23s ago 21m 11.5M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (8m) 5s ago 28m 508M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (7m) 23s ago 26m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (7m) 5s ago 28m 82.4M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (7m) 23s ago 26m 58.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (55s) 5s ago 19m 64.0M - 5.5 9900ed1a6b81 f8bf9272bcd6 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (50s) 23s ago 19m 63.6M - 5.5 9900ed1a6b81 0330629fef30 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (33s) 5s ago 26m 5444k - 1.5.0 0da6a335fe13 5a0918253a79 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (27s) 23s ago 26m 3569k - 1.5.0 0da6a335fe13 aeea6ef1a030 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (5m) 5s ago 25m 218M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (5m) 5s ago 25m 201M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (4m) 5s ago 25m 184M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (4m) 5s ago 24m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (3m) 23s ago 24m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (2m) 23s ago 24m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:09:20.832 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (2m) 23s ago 23m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:09:20.833 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (109s) 23s ago 23m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:09:20.833 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (11s) 5s ago 26m 42.0M - 2.43.0 a07b618ecd1d 12fedced4471 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: pgmap v286: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: pgmap v287: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:09:21.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.136 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:20 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:21.223 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:09:21.224 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: pgmap v286: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:21.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: pgmap v287: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.332 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:20 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": true, 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [ 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "osd", 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "mds", 2024-04-23T01:09:21.546 INFO:teuthology.orchestra.run.smithi121.stdout: "mon", 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr", 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "crash", 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "nfs" 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": "18/23 daemons upgraded", 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:09:21.547 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: Cluster is now healthy 2024-04-23T01:09:21.885 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:21 smithi121 ceph-mon[88653]: from='client.34528 172.21.15.121:0/2101104236' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:21.950 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:09:22.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-23T01:09:22.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:22.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:22.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:22.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-23T01:09:22.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: Cluster is now healthy 2024-04-23T01:09:22.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:21 smithi153 ceph-mon[65847]: from='client.34528 172.21.15.121:0/2101104236' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:22 smithi121 ceph-mon[88653]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:22 smithi121 ceph-mon[88653]: Upgrade: Updating alertmanager.smithi121 2024-04-23T01:09:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:22 smithi121 ceph-mon[88653]: Deploying daemon alertmanager.smithi121 on smithi121 2024-04-23T01:09:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:22 smithi121 ceph-mon[88653]: pgmap v288: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:23.135 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:22 smithi121 ceph-mon[88653]: from='client.34536 172.21.15.121:0/1875584395' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:09:23.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:22 smithi153 ceph-mon[65847]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:23.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:22 smithi153 ceph-mon[65847]: Upgrade: Updating alertmanager.smithi121 2024-04-23T01:09:23.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:22 smithi153 ceph-mon[65847]: Deploying daemon alertmanager.smithi121 on smithi121 2024-04-23T01:09:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:22 smithi153 ceph-mon[65847]: pgmap v288: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:23.331 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:22 smithi153 ceph-mon[65847]: from='client.34536 172.21.15.121:0/1875584395' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:09:23.979 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:23 smithi121 ceph-mon[88653]: pgmap v289: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:24.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:23 smithi153 ceph-mon[65847]: pgmap v289: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:26.184 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:25 smithi121 ceph-mon[88653]: pgmap v290: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:26.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:25 smithi153 ceph-mon[65847]: pgmap v290: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:28.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:28.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:28.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:28 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:28.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:28 smithi121 ceph-mon[88653]: pgmap v291: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:28.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:28.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:28.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:28 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:28.580 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:28 smithi153 ceph-mon[65847]: pgmap v291: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:30.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:29 smithi153 ceph-mon[65847]: pgmap v292: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:30.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:29 smithi121 ceph-mon[88653]: pgmap v292: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 202 B/s rd, 0 op/s 2024-04-23T01:09:32.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:31 smithi153 ceph-mon[65847]: pgmap v293: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:32.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:31 smithi121 ceph-mon[88653]: pgmap v293: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:33.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:33 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:33.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:33 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:34.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:34 smithi121 ceph-mon[88653]: pgmap v294: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:34.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:34 smithi153 ceph-mon[65847]: pgmap v294: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:35.969 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:35.969 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:35 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:36.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:35 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:37.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:36 smithi153 ceph-mon[65847]: pgmap v295: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:37.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:36 smithi121 ceph-mon[88653]: pgmap v295: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:38.074 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.075 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.076 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.076 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.076 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:37 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:38.081 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:37 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:39.030 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:38 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:09:39.030 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:38 smithi121 ceph-mon[88653]: pgmap v296: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:39.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:38 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-23T01:09:39.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:38 smithi153 ceph-mon[65847]: pgmap v296: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:40.045 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:39 smithi121 ceph-mon[88653]: Upgrade: Updating grafana.smithi121 2024-04-23T01:09:40.045 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:39 smithi121 ceph-mon[88653]: Deploying daemon grafana.smithi121 on smithi121 2024-04-23T01:09:40.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:39 smithi153 ceph-mon[65847]: Upgrade: Updating grafana.smithi121 2024-04-23T01:09:40.080 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:39 smithi153 ceph-mon[65847]: Deploying daemon grafana.smithi121 on smithi121 2024-04-23T01:09:41.021 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:40 smithi121 ceph-mon[88653]: pgmap v297: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:41.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:40 smithi153 ceph-mon[65847]: pgmap v297: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:42.009 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:41 smithi121 ceph-mon[88653]: pgmap v298: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:42.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:41 smithi153 ceph-mon[65847]: pgmap v298: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:44.337 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:44.337 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:44.337 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:44 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:44.337 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:44 smithi121 ceph-mon[88653]: pgmap v299: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:44.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:44.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:44.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:44 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:44.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:44 smithi153 ceph-mon[65847]: pgmap v299: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:46.198 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:45 smithi121 ceph-mon[88653]: pgmap v300: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:46.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:45 smithi153 ceph-mon[65847]: pgmap v300: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:48.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:47 smithi153 ceph-mon[65847]: pgmap v301: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:48.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:47 smithi121 ceph-mon[88653]: pgmap v301: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:50.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:50.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:49 smithi153 ceph-mon[65847]: pgmap v302: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:09:50.340 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:49 smithi121 ceph-mon[88653]: pgmap v302: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:52.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:51 smithi153 ceph-mon[65847]: pgmap v303: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:52.384 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:51 smithi121 ceph-mon[88653]: pgmap v303: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:52.399 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (26s) 4s ago 27m 16.4M - 0.25.0 c8568f914cd2 02a08eed3de3 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (6m) 4s ago 27m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (6m) 55s ago 27m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (10s) 4s ago 27m 29.0M - 9.4.7 954c08fa6188 3a7c1718e40c 2024-04-23T01:09:53.057 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (2m) 4s ago 22m 26.9M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (100s) 55s ago 22m 11.5M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (8m) 4s ago 29m 509M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (8m) 55s ago 27m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (7m) 4s ago 29m 80.6M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (7m) 55s ago 26m 58.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (87s) 4s ago 19m 64.3M - 5.5 9900ed1a6b81 f8bf9272bcd6 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (82s) 55s ago 19m 63.6M - 5.5 9900ed1a6b81 0330629fef30 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (65s) 4s ago 27m 8816k - 1.5.0 0da6a335fe13 5a0918253a79 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (59s) 55s ago 26m 3569k - 1.5.0 0da6a335fe13 aeea6ef1a030 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (6m) 4s ago 26m 218M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (5m) 4s ago 25m 201M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (5m) 4s ago 25m 184M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:09:53.058 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (4m) 4s ago 25m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:09:53.059 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (4m) 55s ago 24m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:09:53.059 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (3m) 55s ago 24m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:09:53.059 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (2m) 55s ago 24m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:09:53.059 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (2m) 55s ago 23m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:09:53.059 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (43s) 4s ago 27m 44.8M - 2.43.0 a07b618ecd1d 12fedced4471 2024-04-23T01:09:53.291 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:52 smithi121 ceph-mon[88653]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:53.291 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:52 smithi121 ceph-mon[88653]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:53.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:52 smithi153 ceph-mon[65847]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:53.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:52 smithi153 ceph-mon[65847]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:53.454 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:09:53.455 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": null, 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": false, 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "", 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T01:09:53.813 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": null, 2024-04-23T01:09:53.814 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T01:09:53.814 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:09:53.814 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:09:54.193 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:09:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:54.634 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='client.34552 172.21.15.121:0/2507380525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.635 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: Upgrade: Finalizing container_image settings 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-23T01:09:54.637 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.638 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: Upgrade: Complete! 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.639 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: pgmap v304: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:54.640 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:54 smithi121 ceph-mon[88653]: from='client.34560 172.21.15.121:0/3861905339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.830 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='client.34552 172.21.15.121:0/2507380525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.831 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: Upgrade: Finalizing container_image settings 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-23T01:09:54.832 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.833 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.834 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: Upgrade: Complete! 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.835 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: pgmap v304: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:54.836 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:54 smithi153 ceph-mon[65847]: from='client.34560 172.21.15.121:0/3861905339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:09:56.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:56.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:09:57.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:56 smithi121 ceph-mon[88653]: pgmap v305: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:57.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:56 smithi153 ceph-mon[65847]: pgmap v305: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:09:59.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:09:58 smithi121 ceph-mon[88653]: pgmap v306: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:09:59.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:09:58 smithi153 ceph-mon[65847]: pgmap v306: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:00 smithi121 ceph-mon[88653]: pgmap v307: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:01.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:00 smithi121 ceph-mon[88653]: overall HEALTH_OK 2024-04-23T01:10:01.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:00 smithi153 ceph-mon[65847]: pgmap v307: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:01.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:00 smithi153 ceph-mon[65847]: overall HEALTH_OK 2024-04-23T01:10:03.134 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:02 smithi121 ceph-mon[88653]: pgmap v308: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:03.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:02 smithi153 ceph-mon[65847]: pgmap v308: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:05.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:04 smithi121 ceph-mon[88653]: pgmap v309: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:05.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:05.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:04 smithi153 ceph-mon[65847]: pgmap v309: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:05.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:07.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:06 smithi121 ceph-mon[88653]: pgmap v310: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:07.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:06 smithi153 ceph-mon[65847]: pgmap v310: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:09.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:08 smithi121 ceph-mon[88653]: pgmap v311: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:09.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:08 smithi153 ceph-mon[65847]: pgmap v311: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:11.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:10 smithi121 ceph-mon[88653]: pgmap v312: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:11.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:10 smithi153 ceph-mon[65847]: pgmap v312: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:13.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:12 smithi121 ceph-mon[88653]: pgmap v313: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:13.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:12 smithi153 ceph-mon[65847]: pgmap v313: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:14.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:13 smithi153 ceph-mon[65847]: pgmap v314: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:14.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:13 smithi121 ceph-mon[88653]: pgmap v314: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:16.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:15 smithi153 ceph-mon[65847]: pgmap v315: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:16.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:15 smithi121 ceph-mon[88653]: pgmap v315: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:18.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:17 smithi153 ceph-mon[65847]: pgmap v316: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:18.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:17 smithi121 ceph-mon[88653]: pgmap v316: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:20.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:19 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:20.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:19 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:21.079 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:20 smithi153 ceph-mon[65847]: pgmap v317: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:21.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:20 smithi121 ceph-mon[88653]: pgmap v317: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:22.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:21 smithi153 ceph-mon[65847]: pgmap v318: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:22.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:21 smithi121 ceph-mon[88653]: pgmap v318: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:24.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:23 smithi153 ceph-mon[65847]: pgmap v319: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:24.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:23 smithi121 ceph-mon[88653]: pgmap v319: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:25.129 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:24 smithi121 ceph-mon[88653]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:10:25.294 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph orch ps' 2024-04-23T01:10:25.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:24 smithi153 ceph-mon[65847]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:10:26.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:25 smithi121 ceph-mon[88653]: pgmap v320: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:26.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:25 smithi153 ceph-mon[65847]: pgmap v320: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (61s) 38s ago 28m 16.4M - 0.25.0 c8568f914cd2 02a08eed3de3 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (7m) 38s ago 28m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (7m) 89s ago 27m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (44s) 38s ago 28m 29.0M - 9.4.7 954c08fa6188 3a7c1718e40c 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (2m) 38s ago 22m 26.9M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (2m) 89s ago 22m 11.5M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (9m) 38s ago 29m 509M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (8m) 89s ago 27m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (8m) 38s ago 30m 80.6M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:10:27.464 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (8m) 89s ago 27m 58.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (2m) 38s ago 20m 64.3M - 5.5 9900ed1a6b81 f8bf9272bcd6 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (117s) 89s ago 20m 63.6M - 5.5 9900ed1a6b81 0330629fef30 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (100s) 38s ago 28m 8816k - 1.5.0 0da6a335fe13 5a0918253a79 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (93s) 89s ago 27m 3569k - 1.5.0 0da6a335fe13 aeea6ef1a030 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (7m) 38s ago 26m 218M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (6m) 38s ago 26m 201M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (5m) 38s ago 26m 184M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (5m) 38s ago 25m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (4m) 89s ago 25m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (4m) 89s ago 25m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (3m) 89s ago 24m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (2m) 89s ago 24m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:10:27.465 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (78s) 38s ago 27m 44.8M - 2.43.0 a07b618ecd1d 12fedced4471 2024-04-23T01:10:28.111 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:27 smithi121 ceph-mon[88653]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:10:28.112 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:27 smithi121 ceph-mon[88653]: pgmap v321: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:28.131 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph versions' 2024-04-23T01:10:28.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:27 smithi153 ceph-mon[65847]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:10:28.329 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:27 smithi153 ceph-mon[65847]: pgmap v321: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:30.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:29 smithi121 ceph-mon[88653]: pgmap v322: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:30.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:29 smithi153 ceph-mon[65847]: pgmap v322: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:10:30.398 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:10:30.399 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:10:31.066 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:30 smithi121 ceph-mon[88653]: from='client.34572 172.21.15.121:0/2278343110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:10:31.085 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-23T01:10:31.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:30 smithi153 ceph-mon[65847]: from='client.34572 172.21.15.121:0/2278343110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:10:32.235 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:31 smithi121 ceph-mon[88653]: pgmap v323: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:32.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:31 smithi153 ceph-mon[65847]: pgmap v323: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:33.113 INFO:teuthology.orchestra.run.smithi121.stdout:wait for servicemap items w/ changing names to refresh 2024-04-23T01:10:33.719 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'sleep 60' 2024-04-23T01:10:34.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:33 smithi121 ceph-mon[88653]: pgmap v324: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:34.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:33 smithi153 ceph-mon[65847]: pgmap v324: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:35.070 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:35.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:36.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:35 smithi153 ceph-mon[65847]: pgmap v325: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:36.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:35 smithi121 ceph-mon[88653]: pgmap v325: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:38.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:37 smithi153 ceph-mon[65847]: pgmap v326: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:38.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:37 smithi121 ceph-mon[88653]: pgmap v326: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:40.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:39 smithi153 ceph-mon[65847]: pgmap v327: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:40.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:39 smithi121 ceph-mon[88653]: pgmap v327: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:42.210 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:41 smithi153 ceph-mon[65847]: pgmap v328: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:42.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:41 smithi121 ceph-mon[88653]: pgmap v328: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:44.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:43 smithi153 ceph-mon[65847]: pgmap v329: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:44.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:43 smithi121 ceph-mon[88653]: pgmap v329: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:46.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:45 smithi153 ceph-mon[65847]: pgmap v330: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:46.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:45 smithi121 ceph-mon[88653]: pgmap v330: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:48.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:47 smithi153 ceph-mon[65847]: pgmap v331: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:48.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:47 smithi121 ceph-mon[88653]: pgmap v331: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:50.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:50.133 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:10:51.078 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:50 smithi153 ceph-mon[65847]: pgmap v332: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:51.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:50 smithi121 ceph-mon[88653]: pgmap v332: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:52.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:51 smithi153 ceph-mon[65847]: pgmap v333: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:52.385 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:51 smithi121 ceph-mon[88653]: pgmap v333: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:54.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:53 smithi153 ceph-mon[65847]: pgmap v334: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:54.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:53 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:10:54.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:53 smithi121 ceph-mon[88653]: pgmap v334: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:54.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:53 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:10:55.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:10:55.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:10:55.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:54 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:10:55.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:10:55.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:54 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:10:55.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:10:55.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:10:55.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:54 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:10:55.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:10:55.383 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:54 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:10:56.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:55 smithi153 ceph-mon[65847]: pgmap v335: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:56.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:55 smithi121 ceph-mon[88653]: pgmap v335: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:10:58.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:57 smithi153 ceph-mon[65847]: pgmap v336: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:10:58.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:57 smithi121 ceph-mon[88653]: pgmap v336: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:00.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:10:59 smithi153 ceph-mon[65847]: pgmap v337: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:00.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:10:59 smithi121 ceph-mon[88653]: pgmap v337: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:02.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:01 smithi153 ceph-mon[65847]: pgmap v338: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:02.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:01 smithi121 ceph-mon[88653]: pgmap v338: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:04.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:03 smithi153 ceph-mon[65847]: pgmap v339: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:04.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:03 smithi121 ceph-mon[88653]: pgmap v339: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:05.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:05.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:06.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:05 smithi153 ceph-mon[65847]: pgmap v340: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:06.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:05 smithi121 ceph-mon[88653]: pgmap v340: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:08.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:07 smithi153 ceph-mon[65847]: pgmap v341: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:08.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:07 smithi121 ceph-mon[88653]: pgmap v341: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:10.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:09 smithi153 ceph-mon[65847]: pgmap v342: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:10.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:09 smithi121 ceph-mon[88653]: pgmap v342: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:12.328 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:11 smithi153 ceph-mon[65847]: pgmap v343: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:12.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:11 smithi121 ceph-mon[88653]: pgmap v343: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:14.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:13 smithi153 ceph-mon[65847]: pgmap v344: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:14.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:13 smithi121 ceph-mon[88653]: pgmap v344: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:16.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:15 smithi153 ceph-mon[65847]: pgmap v345: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:16.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:15 smithi121 ceph-mon[88653]: pgmap v345: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:18.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:17 smithi153 ceph-mon[65847]: pgmap v346: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:18.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:17 smithi121 ceph-mon[88653]: pgmap v346: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:20.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:19 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:20.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:19 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:21.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:20 smithi153 ceph-mon[65847]: pgmap v347: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:21.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:20 smithi121 ceph-mon[88653]: pgmap v347: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:22.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:21 smithi153 ceph-mon[65847]: pgmap v348: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:22.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:21 smithi121 ceph-mon[88653]: pgmap v348: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:24.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:23 smithi153 ceph-mon[65847]: pgmap v349: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:24.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:23 smithi121 ceph-mon[88653]: pgmap v349: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:26.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:25 smithi153 ceph-mon[65847]: pgmap v350: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:26.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:25 smithi121 ceph-mon[88653]: pgmap v350: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:28.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:27 smithi153 ceph-mon[65847]: pgmap v351: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:28.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:27 smithi121 ceph-mon[88653]: pgmap v351: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:30.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:29 smithi153 ceph-mon[65847]: pgmap v352: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:30.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:29 smithi121 ceph-mon[88653]: pgmap v352: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:32.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:31 smithi153 ceph-mon[65847]: pgmap v353: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:32.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:31 smithi121 ceph-mon[88653]: pgmap v353: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:34.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:33 smithi153 ceph-mon[65847]: pgmap v354: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:34.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:33 smithi121 ceph-mon[88653]: pgmap v354: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:35.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:34 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:35.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:34 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:36.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:35 smithi153 ceph-mon[65847]: pgmap v355: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:36.333 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:35 smithi121 ceph-mon[88653]: pgmap v355: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:36.357 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph orch ps' 2024-04-23T01:11:38.104 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:37 smithi121 ceph-mon[88653]: pgmap v356: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:38.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:37 smithi153 ceph-mon[65847]: pgmap v356: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:alertmanager.smithi121 smithi121 *:9093,9094 running (2m) 109s ago 29m 16.4M - 0.25.0 c8568f914cd2 02a08eed3de3 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi121 smithi121 running (8m) 109s ago 29m 6647k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 b6c5a766a7b2 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:crash.smithi153 smithi153 running (8m) 2m ago 28m 6651k - 18.2.2-1237-g646b8c7d 9900ed1a6b81 4f68cbd114a4 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:grafana.smithi121 smithi121 *:3000 running (115s) 109s ago 29m 29.0M - 9.4.7 954c08fa6188 3a7c1718e40c 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi121.vupjhr smithi121 running (3m) 109s ago 24m 26.9M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 976c1d238aa5 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mds.foofs.smithi153.uunfav smithi153 running (3m) 2m ago 24m 11.5M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 3ce3839efce8 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi121.otezso smithi121 *:8443,9283,8765 running (10m) 109s ago 31m 509M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 a00c207a32f0 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mgr.smithi153.olhmgd smithi153 *:8443,9283,8765 running (9m) 2m ago 28m 450M - 18.2.2-1237-g646b8c7d 9900ed1a6b81 c633b4abbdde 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi121 smithi121 running (9m) 109s ago 31m 80.6M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a18fb3dc76a1 2024-04-23T01:11:38.571 INFO:teuthology.orchestra.run.smithi121.stdout:mon.smithi153 smithi153 running (9m) 2m ago 28m 58.9M 2048M 18.2.2-1237-g646b8c7d 9900ed1a6b81 135da2e1b3ab 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.0.0.smithi121.aanwei smithi121 *:2049 running (3m) 109s ago 21m 64.3M - 5.5 9900ed1a6b81 f8bf9272bcd6 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:nfs.foo.1.0.smithi153.xxakvf smithi153 *:2049 running (3m) 2m ago 21m 63.6M - 5.5 9900ed1a6b81 0330629fef30 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi121 smithi121 *:9100 running (2m) 109s ago 29m 8816k - 1.5.0 0da6a335fe13 5a0918253a79 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:node-exporter.smithi153 smithi153 *:9100 running (2m) 2m ago 28m 3569k - 1.5.0 0da6a335fe13 aeea6ef1a030 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.0 smithi121 running (8m) 109s ago 28m 218M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 d5d04a0a3e39 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.1 smithi121 running (7m) 109s ago 27m 201M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 0ebdf9aa6cdf 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.2 smithi121 running (6m) 109s ago 27m 184M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 c20ed2b44a08 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.3 smithi121 running (6m) 109s ago 26m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 3958601e8366 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.4 smithi153 running (5m) 2m ago 26m 215M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 8c2d0e2f414e 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.5 smithi153 running (5m) 2m ago 26m 209M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 5ab677d63a56 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.6 smithi153 running (4m) 2m ago 25m 200M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 89858230e179 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:osd.7 smithi153 running (4m) 2m ago 25m 239M 4096M 18.2.2-1237-g646b8c7d 9900ed1a6b81 a0b29f29af71 2024-04-23T01:11:38.572 INFO:teuthology.orchestra.run.smithi121.stdout:prometheus.smithi121 smithi121 *:9095 running (2m) 109s ago 28m 44.8M - 2.43.0 a07b618ecd1d 12fedced4471 2024-04-23T01:11:39.187 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph orch upgrade status' 2024-04-23T01:11:39.213 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:38 smithi121 ceph-mon[88653]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:11:39.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:38 smithi153 ceph-mon[65847]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:11:40.296 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:39 smithi121 ceph-mon[88653]: pgmap v357: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:40.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:39 smithi153 ceph-mon[65847]: pgmap v357: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:41.312 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:11:41.312 INFO:teuthology.orchestra.run.smithi121.stdout: "target_image": null, 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "in_progress": false, 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "which": "", 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "services_complete": [], 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "progress": null, 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "message": "", 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout: "is_paused": false 2024-04-23T01:11:41.313 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:11:41.995 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph health detail' 2024-04-23T01:11:42.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:41 smithi153 ceph-mon[65847]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:11:42.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:41 smithi153 ceph-mon[65847]: pgmap v358: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:42.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:41 smithi121 ceph-mon[88653]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:11:42.382 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:41 smithi121 ceph-mon[88653]: pgmap v358: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:44.132 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:43 smithi121 ceph-mon[88653]: pgmap v359: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:44.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:43 smithi153 ceph-mon[65847]: pgmap v359: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:44.395 INFO:teuthology.orchestra.run.smithi121.stdout:HEALTH_OK 2024-04-23T01:11:45.128 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:44 smithi121 ceph-mon[88653]: from='client.34584 172.21.15.121:0/2470639993' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:11:45.208 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph versions' 2024-04-23T01:11:45.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:44 smithi153 ceph-mon[65847]: from='client.34584 172.21.15.121:0/2470639993' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-23T01:11:46.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:45 smithi121 ceph-mon[88653]: pgmap v360: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:46.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:45 smithi153 ceph-mon[65847]: pgmap v360: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:47.358 INFO:teuthology.orchestra.run.smithi121.stdout:{ 2024-04-23T01:11:47.358 INFO:teuthology.orchestra.run.smithi121.stdout: "mon": { 2024-04-23T01:11:47.358 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "mgr": { 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "osd": { 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 8 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "mds": { 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 2 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "overall": { 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)": 14 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:11:47.359 INFO:teuthology.orchestra.run.smithi121.stdout:} 2024-04-23T01:11:47.806 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:47 smithi121 ceph-mon[88653]: from='client.34588 172.21.15.121:0/1423130539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:47.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:47 smithi153 ceph-mon[65847]: from='client.34588 172.21.15.121:0/1423130539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:48.100 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-23T01:11:48.803 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:48 smithi121 ceph-mon[88653]: pgmap v361: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:48.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:48 smithi153 ceph-mon[65847]: pgmap v361: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:50.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:49 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:50.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:49 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:11:50.251 INFO:teuthology.orchestra.run.smithi121.stdout:true 2024-04-23T01:11:50.974 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:50 smithi121 ceph-mon[88653]: pgmap v362: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:50.974 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:50 smithi121 ceph-mon[88653]: from='client.34592 172.21.15.121:0/961803344' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:50.993 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -e sha1=646b8c7d293d8ec31a40c69f7b3237b19c1be24d -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-23T01:11:51.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:50 smithi153 ceph-mon[65847]: pgmap v362: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:51.077 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:50 smithi153 ceph-mon[65847]: from='client.34592 172.21.15.121:0/961803344' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:52.743 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:52 smithi121 ceph-mon[88653]: pgmap v363: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:52.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:52 smithi153 ceph-mon[65847]: pgmap v363: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:53.265 INFO:teuthology.orchestra.run.smithi121.stdout: "ceph version 18.2.2-1237-g646b8c7d (646b8c7d293d8ec31a40c69f7b3237b19c1be24d) reef (stable)" 2024-04-23T01:11:53.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:53 smithi121 ceph-mon[88653]: from='client.34596 172.21.15.121:0/295029913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:53.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:53 smithi153 ceph-mon[65847]: from='client.34596 172.21.15.121:0/295029913' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-23T01:11:53.939 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-23T01:11:53.940 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-23T01:11:53.941 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-23T01:11:54.631 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:54 smithi121 ceph-mon[88653]: pgmap v364: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:54.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:54 smithi153 ceph-mon[65847]: pgmap v364: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:55.744 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:11:55.745 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.745 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.745 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.745 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:55 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-23T01:11:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:55.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:55 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:56.205 INFO:teuthology.orchestra.run.smithi121.stdout: 2024-04-23T01:11:56.205 INFO:teuthology.orchestra.run.smithi121.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-23T00:41:15.984619Z", "last_refresh": "2024-04-23T01:09:48.952054Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-23T00:41:02.841325Z", "last_refresh": "2024-04-23T01:08:57.513708Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-23T00:41:12.995137Z", "last_refresh": "2024-04-23T01:09:48.952180Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-23T00:47:26.794077Z", "last_refresh": "2024-04-23T01:08:57.514205Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-23T00:41:00.650258Z", "last_refresh": "2024-04-23T01:08:57.513788Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi121:172.21.15.121=smithi121", "smithi153:172.21.15.153=smithi153"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-23T00:41:49.481977Z", "last_refresh": "2024-04-23T01:08:57.513850Z", "running": 2, "size": 2}}, {"events": ["2024-04-23T01:11:55.706620Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "status": {"created": "2024-04-23T00:50:06.256403Z", "last_refresh": "2024-04-23T01:08:57.514279Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-23T00:41:14.603199Z", "last_refresh": "2024-04-23T01:08:57.513911Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "9900ed1a6b81aaf043a19454fc819f6cb21ac0421c7c720b31d8fae6faebb644", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:417b28a7fda24d3105eeb9bae56a845b9ffa154499a85601880a5fd035847210", "last_refresh": "2024-04-23T01:08:57.513970Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-23T00:41:10.868310Z", "last_refresh": "2024-04-23T01:09:48.952359Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-23T01:11:56.562 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:11:56.562 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:11:56.562 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:11:56.562 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:56.562 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:56.563 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:56 smithi121 ceph-mon[88653]: pgmap v365: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:56.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-23T01:11:56.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-23T01:11:56.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-04-23T01:11:56.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:56.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' 2024-04-23T01:11:56.827 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:56 smithi153 ceph-mon[65847]: pgmap v365: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-23T01:11:56.845 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-23T01:11:56.845 DEBUG:teuthology.parallel:result is None 2024-04-23T01:11:56.846 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-23T01:11:56.857 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T01:11:56.858 DEBUG:teuthology.orchestra.run.smithi121:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-23T01:11:56.888 INFO:teuthology.orchestra.run.smithi121.stderr:+ umount /mnt/foo 2024-04-23T01:11:56.902 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-23T01:11:56.912 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T01:11:56.912 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-23T01:11:58.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:57 smithi153 ceph-mon[65847]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T01:11:58.119 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:57 smithi121 ceph-mon[88653]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-23T01:11:59.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:58 smithi153 ceph-mon[65847]: pgmap v366: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:59.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:58 smithi121 ceph-mon[88653]: pgmap v366: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:11:59.186 INFO:teuthology.orchestra.run.smithi121.stdout: "foo" 2024-04-23T01:11:59.875 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-23T01:12:00.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:59 smithi121 ceph-mon[88653]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:12:00.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:11:59 smithi121 ceph-mon[88653]: pgmap v367: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:00.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:59 smithi153 ceph-mon[65847]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:12:00.330 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:11:59 smithi153 ceph-mon[65847]: pgmap v367: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:02.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:01 smithi121 ceph-mon[88653]: pgmap v368: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout:[ 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: { 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "access_type": "RW", 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "clients": [], 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "cluster_id": "foo", 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "export_id": 1, 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "fsal": { 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "fs_name": "foofs", 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "name": "CEPH", 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "user_id": "nfs.foo.1" 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: }, 2024-04-23T01:12:02.180 INFO:teuthology.orchestra.run.smithi121.stdout: "path": "/", 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "protocols": [ 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: 4 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: ], 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "pseudo": "/fake", 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "security_label": true, 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "squash": "no_root_squash", 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "transports": [ 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: "TCP" 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: ] 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout: } 2024-04-23T01:12:02.181 INFO:teuthology.orchestra.run.smithi121.stdout:] 2024-04-23T01:12:02.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:01 smithi153 ceph-mon[65847]: pgmap v368: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:02.985 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-23T01:12:03.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:02 smithi153 ceph-mon[65847]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:12:03.377 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:02 smithi121 ceph-mon[88653]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-23T01:12:04.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:03 smithi153 ceph-mon[65847]: pgmap v369: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:04.369 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:03 smithi121 ceph-mon[88653]: pgmap v369: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-23T01:12:04.892 INFO:teuthology.orchestra.run.smithi121.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-23T01:12:04.892 INFO:teuthology.orchestra.run.smithi121.stdout:foo grace 2024-04-23T01:12:04.893 INFO:teuthology.orchestra.run.smithi121.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-23T01:12:04.893 INFO:teuthology.orchestra.run.smithi121.stdout:foo conf-nfs.foo 2024-04-23T01:12:04.893 INFO:teuthology.orchestra.run.smithi121.stdout:foo export-1 2024-04-23T01:12:05.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:04 smithi153 ceph-mon[65847]: mgrmap e47: smithi121.otezso(active, since 9m), standbys: smithi153.olhmgd 2024-04-23T01:12:05.327 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:04 smithi153 ceph-mon[65847]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:12:05.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:04 smithi121 ceph-mon[88653]: mgrmap e47: smithi121.otezso(active, since 9m), standbys: smithi153.olhmgd 2024-04-23T01:12:05.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:04 smithi121 ceph-mon[88653]: from='mgr.34104 172.21.15.121:0/3623621187' entity='mgr.smithi121.otezso' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-23T01:12:05.487 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 004558ac-010a-11ef-bc93-c7b262605968 -- bash -c 'ceph config get mgr mgr/cephadm/migration_current | grep 6' 2024-04-23T01:12:06.326 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:05 smithi153 ceph-mon[65847]: pgmap v370: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-23T01:12:06.381 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:05 smithi121 ceph-mon[88653]: pgmap v370: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-23T01:12:07.639 INFO:teuthology.orchestra.run.smithi121.stdout:6 2024-04-23T01:12:08.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:07 smithi153 ceph-mon[65847]: from='client.34622 172.21.15.121:0/252213263' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-23T01:12:08.131 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:07 smithi121 ceph-mon[88653]: from='client.34622 172.21.15.121:0/252213263' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-23T01:12:08.555 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-23T01:12:08.572 INFO:tasks.cephadm:Teardown begin 2024-04-23T01:12:08.572 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-23T01:12:08.605 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-23T01:12:08.644 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-23T01:12:08.644 DEBUG:teuthology.orchestra.run.smithi121:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-23T01:12:08.665 DEBUG:teuthology.orchestra.run.smithi153:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-23T01:12:08.704 INFO:tasks.cephadm:Stopping all daemons... 2024-04-23T01:12:08.705 INFO:tasks.cephadm.mon.smithi121:Stopping mon.smithi121... 2024-04-23T01:12:08.705 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121 2024-04-23T01:12:08.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:08 smithi121 ceph-mon[88653]: pgmap v371: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:12:08.881 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:08 smithi121 systemd[1]: Stopping Ceph mon.smithi121 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:09.076 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:08 smithi153 ceph-mon[65847]: pgmap v371: 129 pgs: 129 active+clean; 218 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-23T01:12:09.323 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi121[88630]: 2024-04-23T01:12:08.969+0000 7f76d1b60700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi121 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:09.323 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:08 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi121[88630]: 2024-04-23T01:12:08.969+0000 7f76d1b60700 -1 mon.smithi121@0(leader) e3 *** Got Signal Terminated *** 2024-04-23T01:12:09.636 INFO:journalctl@ceph.mon.smithi121.smithi121.stdout:Apr 23 01:12:09 smithi121 bash[149094]: ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi121 2024-04-23T01:12:09.817 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi121.service' 2024-04-23T01:12:09.872 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:09.872 INFO:tasks.cephadm.mon.smithi121:Stopped mon.smithi121 2024-04-23T01:12:09.873 INFO:tasks.cephadm.mon.smithi153:Stopping mon.smithi153... 2024-04-23T01:12:09.873 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi153 2024-04-23T01:12:10.178 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:09 smithi153 systemd[1]: Stopping Ceph mon.smithi153 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:10.488 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi153[65824]: 2024-04-23T01:12:10.171+0000 7fbe16337700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi153 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:10.488 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi153[65824]: 2024-04-23T01:12:10.171+0000 7fbe16337700 -1 mon.smithi153@1(peon) e3 *** Got Signal Terminated *** 2024-04-23T01:12:10.826 INFO:journalctl@ceph.mon.smithi153.smithi153.stdout:Apr 23 01:12:10 smithi153 bash[97085]: ceph-004558ac-010a-11ef-bc93-c7b262605968-mon-smithi153 2024-04-23T01:12:11.052 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@mon.smithi153.service' 2024-04-23T01:12:11.096 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:11.097 INFO:tasks.cephadm.mon.smithi153:Stopped mon.smithi153 2024-04-23T01:12:11.097 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-23T01:12:11.097 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.0 2024-04-23T01:12:11.381 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:12:11 smithi121 systemd[1]: Stopping Ceph osd.0 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:11.381 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:12:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:12:11.367+0000 7fdbe3598700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:11.381 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:12:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:12:11.367+0000 7fdbe3598700 -1 osd.0 120 *** Got signal Terminated *** 2024-04-23T01:12:11.381 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:12:11 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0[95715]: 2024-04-23T01:12:11.367+0000 7fdbe3598700 -1 osd.0 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:17.381 INFO:journalctl@ceph.osd.0.smithi121.stdout:Apr 23 01:12:17 smithi121 bash[149251]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-0 2024-04-23T01:12:19.009 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.0.service' 2024-04-23T01:12:19.050 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:19.050 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-23T01:12:19.051 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-23T01:12:19.051 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.1 2024-04-23T01:12:19.331 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:12:19 smithi121 systemd[1]: Stopping Ceph osd.1 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:19.631 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:12:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:12:19.325+0000 7f54bd2bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:19.631 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:12:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:12:19.325+0000 7f54bd2bd700 -1 osd.1 120 *** Got signal Terminated *** 2024-04-23T01:12:19.631 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:12:19 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1[99278]: 2024-04-23T01:12:19.325+0000 7f54bd2bd700 -1 osd.1 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:25.380 INFO:journalctl@ceph.osd.1.smithi121.stdout:Apr 23 01:12:25 smithi121 bash[149553]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-1 2024-04-23T01:12:26.919 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.1.service' 2024-04-23T01:12:26.960 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:26.961 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-23T01:12:26.961 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-23T01:12:26.961 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.2 2024-04-23T01:12:27.256 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:12:27 smithi121 systemd[1]: Stopping Ceph osd.2 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:27.630 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:12:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:12:27.250+0000 7f3c35f53700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:27.631 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:12:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:12:27.250+0000 7f3c35f53700 -1 osd.2 120 *** Got signal Terminated *** 2024-04-23T01:12:27.631 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:12:27 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2[102721]: 2024-04-23T01:12:27.250+0000 7f3c35f53700 -1 osd.2 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:33.381 INFO:journalctl@ceph.osd.2.smithi121.stdout:Apr 23 01:12:33 smithi121 bash[149820]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-2 2024-04-23T01:12:35.027 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.2.service' 2024-04-23T01:12:35.067 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:35.068 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-23T01:12:35.068 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-23T01:12:35.068 DEBUG:teuthology.orchestra.run.smithi121:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.3 2024-04-23T01:12:35.378 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:35 smithi121 systemd[1]: Stopping Ceph osd.3 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:35.630 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:35.372+0000 7fe656363700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:35.630 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:35.372+0000 7fe656363700 -1 osd.3 120 *** Got signal Terminated *** 2024-04-23T01:12:35.631 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:35 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:35.372+0000 7fe656363700 -1 osd.3 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:37.631 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:37 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:37.221+0000 7fe64e951700 -1 osd.3 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.686009+0000 front 2024-04-23T01:12:15.686090+0000 (oldest deadline 2024-04-23T01:12:36.786189+0000) 2024-04-23T01:12:38.630 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:38 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:38.258+0000 7fe64e951700 -1 osd.3 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.686009+0000 front 2024-04-23T01:12:15.686090+0000 (oldest deadline 2024-04-23T01:12:36.786189+0000) 2024-04-23T01:12:39.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:39 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:39.205+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:39.630 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:39 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:39.302+0000 7fe64e951700 -1 osd.3 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.686009+0000 front 2024-04-23T01:12:15.686090+0000 (oldest deadline 2024-04-23T01:12:36.786189+0000) 2024-04-23T01:12:40.576 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:40.117+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:40.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:40 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:40.174+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:40.630 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:40 smithi121 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3[106150]: 2024-04-23T01:12:40.349+0000 7fe64e951700 -1 osd.3 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.686009+0000 front 2024-04-23T01:12:15.686090+0000 (oldest deadline 2024-04-23T01:12:36.786189+0000) 2024-04-23T01:12:41.380 INFO:journalctl@ceph.osd.3.smithi121.stdout:Apr 23 01:12:41 smithi121 bash[150129]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-3 2024-04-23T01:12:41.542 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:41.118+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:41.543 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:41.140+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:41.826 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:41 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:41.538+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:42.572 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:42.148+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:42.573 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:42.116+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:42.573 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:42.208+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:42.826 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:42 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:42.568+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:42.955 DEBUG:teuthology.orchestra.run.smithi121:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.3.service' 2024-04-23T01:12:42.997 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:42.997 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-23T01:12:42.997 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-23T01:12:42.997 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.4 2024-04-23T01:12:43.302 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:43.161+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:43.303 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:43.155+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:43.303 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:43.207+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:43.303 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:43 smithi153 systemd[1]: Stopping Ceph osd.4 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:43.303 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:43.298+0000 7f7727d08700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:43.576 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:43.298+0000 7f7727d08700 -1 osd.4 120 *** Got signal Terminated *** 2024-04-23T01:12:43.576 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:43.298+0000 7f7727d08700 -1 osd.4 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:43.576 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:43 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:43.542+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:44.569 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:44.133+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:44.569 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:44.127+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:44.570 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:44.209+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:44.826 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:44 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:44.565+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:45.576 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:45.165+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:45.577 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:45.254+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:45.577 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:45.161+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:46.076 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:45.584+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:46.076 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:45 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:45.584+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:20.643860+0000 front 2024-04-23T01:12:20.643931+0000 (oldest deadline 2024-04-23T01:12:44.743733+0000) 2024-04-23T01:12:46.559 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:46.126+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:46.560 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:46.128+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:46.560 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:46.254+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:46.826 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:46.555+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:46.826 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:46 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:46.555+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:20.643860+0000 front 2024-04-23T01:12:20.643931+0000 (oldest deadline 2024-04-23T01:12:44.743733+0000) 2024-04-23T01:12:47.575 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:47.147+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:47.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:47.298+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:47.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:47.298+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:47.576 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:47.147+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:48.075 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:47.592+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.943410+0000 front 2024-04-23T01:12:15.943531+0000 (oldest deadline 2024-04-23T01:12:40.643287+0000) 2024-04-23T01:12:48.076 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:47 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4[71759]: 2024-04-23T01:12:47.593+0000 7f77242fe700 -1 osd.4 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:20.643860+0000 front 2024-04-23T01:12:20.643931+0000 (oldest deadline 2024-04-23T01:12:44.743733+0000) 2024-04-23T01:12:48.576 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:48.135+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:48.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:48.345+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:48.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:48.345+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:48.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:48.160+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:48.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:48 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:48.160+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:49.326 INFO:journalctl@ceph.osd.4.smithi153.stdout:Apr 23 01:12:49 smithi153 bash[97311]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-4 2024-04-23T01:12:49.326 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:49.193+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:49.326 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:49.193+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:49.327 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:49.098+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:49.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:49.341+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:49.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:49 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:49.341+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:50.301 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:50.224+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:50.301 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:50.224+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:50.301 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:50.083+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:50.301 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:50.083+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:50.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:50.296+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:50.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:50 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:50.296+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:50.979 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.4.service' 2024-04-23T01:12:51.020 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:51.020 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-23T01:12:51.020 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-23T01:12:51.020 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.5 2024-04-23T01:12:51.321 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:51.044+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:51.321 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:51.044+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:51.322 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:51.255+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:51.322 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:51.255+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:51.322 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 systemd[1]: Stopping Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:51.322 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:51.208+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:51.322 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:51.208+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:51.575 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:51.317+0000 7fe6d6dc6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:51.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:51.317+0000 7fe6d6dc6700 -1 osd.5 120 *** Got signal Terminated *** 2024-04-23T01:12:51.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:51 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:51.317+0000 7fe6d6dc6700 -1 osd.5 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:52.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:52.033+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:52.309 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:52.033+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:52.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:52.256+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:52.310 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:52.256+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:52.575 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:52.305+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:52.575 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:52.305+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:53.271 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:52.987+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:53.271 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:52 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:52.987+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:53.575 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:53.267+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:53.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:53.267+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:53.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:53.355+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:53.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:53.355+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:54.285 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:53.956+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:54.285 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:53 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:53.956+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:54.575 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:54.281+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:54.576 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:54.281+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:54.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:54.318+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:54.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:54.318+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:55.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:54.937+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:55.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:54.937+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:55.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:54 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:54.937+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:12:55.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:55.359+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:55.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:55.359+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:55.826 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:55.331+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.887625+0000 front 2024-04-23T01:12:15.887390+0000 (oldest deadline 2024-04-23T01:12:38.787523+0000) 2024-04-23T01:12:55.826 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:55.331+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.488361+0000 front 2024-04-23T01:12:23.488465+0000 (oldest deadline 2024-04-23T01:12:47.588456+0000) 2024-04-23T01:12:55.826 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5[74484]: 2024-04-23T01:12:55.331+0000 7fe6d33bc700 -1 osd.5 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:31.589891+0000 front 2024-04-23T01:12:31.589983+0000 (oldest deadline 2024-04-23T01:12:54.489911+0000) 2024-04-23T01:12:56.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:55.931+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:56.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:55.931+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:56.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:55 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:55.931+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:12:56.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:56.372+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:56.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:56.372+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:56.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:56.372+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:12:57.264 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:56.964+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:57.264 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:56.964+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:57.264 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:56 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:56.964+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:12:57.578 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:57 smithi153 bash[97579]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-5 2024-04-23T01:12:57.578 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:57.359+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:57.578 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:57.359+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:57.578 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:57.359+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:12:58.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:57.976+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:58.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:57.976+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:58.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:57 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:57.976+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:12:58.703 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:58.408+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:58.703 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:58.408+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:58.704 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:58.408+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:12:58.947 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.5.service' 2024-04-23T01:12:58.974 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:58 smithi153 systemd[1]: ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.5.service: Succeeded. 2024-04-23T01:12:58.974 INFO:journalctl@ceph.osd.5.smithi153.stdout:Apr 23 01:12:58 smithi153 systemd[1]: Stopped Ceph osd.5 for 004558ac-010a-11ef-bc93-c7b262605968. 2024-04-23T01:12:58.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:12:58.988 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-23T01:12:58.988 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-23T01:12:58.988 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.6 2024-04-23T01:12:59.293 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:58.989+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:12:59.293 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:58.989+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:12:59.293 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:58 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:58.989+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:12:59.293 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 systemd[1]: Stopping Ceph osd.6 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:12:59.293 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.289+0000 7f84b7190700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:12:59.294 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.289+0000 7f84b7190700 -1 osd.6 120 *** Got signal Terminated *** 2024-04-23T01:12:59.575 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.289+0000 7f84b7190700 -1 osd.6 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:12:59.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:59.444+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:12:59.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:59.444+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:12:59.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:12:59.444+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:00.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:13:00.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:13:00.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:12:59 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:12:59.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:13:00.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:00.450+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:00.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:00.450+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:00.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:00.450+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:01.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:00.913+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:13:01.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:00.913+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:13:01.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:00 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:00.913+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:13:01.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:01.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:01.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:01.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:01.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:01.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:02.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:01.963+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:13:02.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:01.963+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:13:02.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:01.963+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:13:02.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:01 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:01.963+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:39.572759+0000 front 2024-04-23T01:12:39.572829+0000 (oldest deadline 2024-04-23T01:13:01.872617+0000) 2024-04-23T01:13:02.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:02.424+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:02.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:02.424+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:02.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:02.424+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:03.325 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:02.973+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:13:03.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:02.973+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:13:03.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:02.973+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:13:03.326 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:02 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:02.973+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:39.572759+0000 front 2024-04-23T01:12:39.572829+0000 (oldest deadline 2024-04-23T01:13:01.872617+0000) 2024-04-23T01:13:03.814 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:03.401+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:03.814 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:03.401+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:03.814 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:03.401+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:04.075 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:03.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:14.669105+0000 front 2024-04-23T01:12:14.669191+0000 (oldest deadline 2024-04-23T01:12:39.368964+0000) 2024-04-23T01:13:04.075 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:03.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.970399+0000 front 2024-04-23T01:12:23.970345+0000 (oldest deadline 2024-04-23T01:12:49.870274+0000) 2024-04-23T01:13:04.075 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:03.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:29.870906+0000 front 2024-04-23T01:12:29.870950+0000 (oldest deadline 2024-04-23T01:12:53.970745+0000) 2024-04-23T01:13:04.076 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:03 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6[77206]: 2024-04-23T01:13:03.949+0000 7f84af77e700 -1 osd.6 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:39.572759+0000 front 2024-04-23T01:12:39.572829+0000 (oldest deadline 2024-04-23T01:13:01.872617+0000) 2024-04-23T01:13:04.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:04.421+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:04.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:04.421+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:04.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:04 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:04.421+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:05.327 INFO:journalctl@ceph.osd.6.smithi153.stdout:Apr 23 01:13:05 smithi153 bash[97880]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-6 2024-04-23T01:13:05.801 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:05.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:05.801 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:05.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:05.801 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:05.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:05.801 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:05 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:05.412+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:06.524 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:06.368+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:06.524 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:06.368+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:06.524 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:06.368+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:06.524 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:06 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:06.368+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:06.819 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.6.service' 2024-04-23T01:13:06.863 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:13:06.918 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-23T01:13:06.918 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-23T01:13:06.919 DEBUG:teuthology.orchestra.run.smithi153:> sudo systemctl stop ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.7 2024-04-23T01:13:07.075 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:06 smithi153 systemd[1]: Stopping Ceph osd.7 for 004558ac-010a-11ef-bc93-c7b262605968... 2024-04-23T01:13:07.575 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.167+0000 7fac0f7bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-23T01:13:07.575 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.167+0000 7fac0f7bc700 -1 osd.7 120 *** Got signal Terminated *** 2024-04-23T01:13:07.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.168+0000 7fac0f7bc700 -1 osd.7 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-23T01:13:07.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.390+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:07.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.390+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:07.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.390+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:07.576 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:07 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:07.390+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:08.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:08.395+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:08.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:08.395+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:08.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:08.395+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:08.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:08 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:08.395+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:09.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:09.382+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:09.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:09.382+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:09.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:09.382+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:09.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:09 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:09.382+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:10.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:10.351+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:10.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:10.351+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:10.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:10.351+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:10.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:10 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:10.351+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:11.825 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:11.375+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6806 osd.0 since back 2024-04-23T01:12:15.538001+0000 front 2024-04-23T01:12:15.537973+0000 (oldest deadline 2024-04-23T01:12:41.437917+0000) 2024-04-23T01:13:11.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:11.375+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6814 osd.1 since back 2024-04-23T01:12:23.739032+0000 front 2024-04-23T01:12:23.739092+0000 (oldest deadline 2024-04-23T01:12:47.238907+0000) 2024-04-23T01:13:11.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:11.375+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6822 osd.2 since back 2024-04-23T01:12:30.139707+0000 front 2024-04-23T01:12:30.139692+0000 (oldest deadline 2024-04-23T01:12:56.039689+0000) 2024-04-23T01:13:11.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:11.375+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.121:6830 osd.3 since back 2024-04-23T01:12:36.040187+0000 front 2024-04-23T01:12:36.040124+0000 (oldest deadline 2024-04-23T01:13:04.840568+0000) 2024-04-23T01:13:11.826 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:11 smithi153 ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7[79893]: 2024-04-23T01:13:11.375+0000 7fac0bdb2700 -1 osd.7 120 heartbeat_check: no reply from 172.21.15.153:6804 osd.4 since back 2024-04-23T01:12:44.841041+0000 front 2024-04-23T01:12:44.841073+0000 (oldest deadline 2024-04-23T01:13:10.741103+0000) 2024-04-23T01:13:13.325 INFO:journalctl@ceph.osd.7.smithi153.stdout:Apr 23 01:13:13 smithi153 bash[98145]: ceph-004558ac-010a-11ef-bc93-c7b262605968-osd-7 2024-04-23T01:13:14.854 DEBUG:teuthology.orchestra.run.smithi153:> sudo pkill -f 'journalctl -f -n 0 -u ceph-004558ac-010a-11ef-bc93-c7b262605968@osd.7.service' 2024-04-23T01:13:14.897 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-23T01:13:14.897 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-23T01:13:14.898 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 004558ac-010a-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-23T01:13:15.077 INFO:teuthology.orchestra.run.smithi121.stderr:usage: cephadm [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR] 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr: [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR] 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr: [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT] 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr: [--retry RETRY] [--env ENV] [--no-container-init] 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr: {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install,registry-login,gather-facts,exporter,host-maintenance,verify-prereqs} 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr: ... 2024-04-23T01:13:15.078 INFO:teuthology.orchestra.run.smithi121.stderr:cephadm: error: unrecognized arguments: --keep-logs 2024-04-23T01:13:15.093 DEBUG:teuthology.orchestra.run:got remote process result: 2 2024-04-23T01:13:15.093 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 004558ac-010a-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr:usage: cephadm [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR] 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr: [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR] 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr: [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT] 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr: [--retry RETRY] [--env ENV] [--no-container-init] 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr: {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install,registry-login,gather-facts,exporter,host-maintenance,verify-prereqs} 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr: ... 2024-04-23T01:13:15.268 INFO:teuthology.orchestra.run.smithi153.stderr:cephadm: error: unrecognized arguments: --keep-logs 2024-04-23T01:13:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 2 2024-04-23T01:13:15.285 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-23T01:13:15.311 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-23T01:13:15.338 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-23T01:13:15.340 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi121/crash 2024-04-23T01:13:15.340 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/crash -- . 2024-04-23T01:13:15.386 DEBUG:teuthology.misc:Transferring archived files from smithi153:/var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi153/crash 2024-04-23T01:13:15.387 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /var/lib/ceph/004558ac-010a-11ef-bc93-c7b262605968/crash -- . 2024-04-23T01:13:15.426 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-23T01:13:15.426 DEBUG:teuthology.orchestra.run.smithi121:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-23T01:13:15.456 INFO:teuthology.orchestra.run.smithi121.stderr:grep: /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968/ceph.log: No such file or directory 2024-04-23T01:13:15.457 INFO:tasks.cephadm:Compressing logs... 2024-04-23T01:13:15.457 DEBUG:teuthology.orchestra.run.smithi121:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-23T01:13:15.500 DEBUG:teuthology.orchestra.run.smithi153:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-23T01:13:15.524 INFO:teuthology.orchestra.run.smithi121.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-23T01:13:15.526 INFO:teuthology.orchestra.run.smithi153.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-23T01:14:00.762 INFO:teuthology.orchestra.run.smithi121.stderr:gzip: /var/log/ceph/004558ac-010a-11ef-bc93-c7b262605968/ceph-mgr.smithi121.otezso.log: file size changed while zipping 2024-04-23T01:14:32.603 INFO:tasks.cephadm:Archiving logs... 2024-04-23T01:14:32.604 DEBUG:teuthology.misc:Transferring archived files from smithi121:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi121/log 2024-04-23T01:14:32.605 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-23T01:14:40.281 DEBUG:teuthology.misc:Transferring archived files from smithi153:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi153/log 2024-04-23T01:14:40.283 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-23T01:14:49.105 INFO:tasks.cephadm:Removing cluster... 2024-04-23T01:14:49.105 DEBUG:teuthology.orchestra.run.smithi121:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 004558ac-010a-11ef-bc93-c7b262605968 --force 2024-04-23T01:17:44.017 DEBUG:teuthology.orchestra.run.smithi153:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 004558ac-010a-11ef-bc93-c7b262605968 --force 2024-04-23T01:18:04.816 INFO:tasks.cephadm:Removing cephadm ... 2024-04-23T01:18:04.816 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-23T01:18:04.840 DEBUG:teuthology.orchestra.run.smithi153:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-23T01:18:04.859 INFO:tasks.cephadm:Teardown complete 2024-04-23T01:18:04.859 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-23T01:18:04.872 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-23T01:18:04.872 DEBUG:teuthology.orchestra.run.smithi121:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-23T01:18:04.883 DEBUG:teuthology.orchestra.run.smithi153:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-23T01:18:04.900 INFO:teuthology.orchestra.run.smithi121.stderr:bash: ntpq: command not found 2024-04-23T01:18:04.905 INFO:teuthology.orchestra.run.smithi121.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-23T01:18:04.905 INFO:teuthology.orchestra.run.smithi121.stdout:=============================================================================== 2024-04-23T01:18:04.905 INFO:teuthology.orchestra.run.smithi121.stdout:^- hv01.front.sepia.ceph.com 2 8 377 166 +1016us[+1119us] +/- 587ms 2024-04-23T01:18:04.905 INFO:teuthology.orchestra.run.smithi121.stdout:^* hv02.front.sepia.ceph.com 3 6 377 42 -508us[ -403us] +/- 55ms 2024-04-23T01:18:04.906 INFO:teuthology.orchestra.run.smithi121.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 103 +663us[ +767us] +/- 65ms 2024-04-23T01:18:04.906 INFO:teuthology.orchestra.run.smithi121.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T01:18:04.916 INFO:teuthology.orchestra.run.smithi153.stderr:bash: ntpq: command not found 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:=============================================================================== 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:^- hv01.front.sepia.ceph.com 2 6 377 34 +1039us[+1044us] +/- 589ms 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:^* hv02.front.sepia.ceph.com 3 6 377 34 -550us[ -544us] +/- 56ms 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 37 +645us[ +650us] +/- 66ms 2024-04-23T01:18:04.922 INFO:teuthology.orchestra.run.smithi153.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-23T01:18:04.923 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-23T01:18:04.937 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-23T01:18:04.939 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-23T01:18:04.959 DEBUG:teuthology.orchestra.run.smithi121:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-23T01:18:05.035 DEBUG:teuthology.orchestra.run.smithi153:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-23T01:18:05.103 DEBUG:teuthology.orchestra.run.smithi121:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-23T01:18:05.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T01:18:05.135 DEBUG:teuthology.orchestra.run.smithi153:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-23T01:18:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T01:18:05.164 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-23T01:18:05.175 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-23T01:18:05.204 INFO:teuthology.task.internal:Duration was 2660.874052 seconds 2024-04-23T01:18:05.205 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-23T01:18:05.234 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-23T01:18:05.235 DEBUG:teuthology.orchestra.run.smithi121:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-23T01:18:05.237 DEBUG:teuthology.orchestra.run.smithi153:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-23T01:18:05.281 INFO:teuthology.orchestra.run.smithi121.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-23T01:18:05.283 INFO:teuthology.orchestra.run.smithi153.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-23T01:18:05.639 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-23T01:18:05.639 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi121.front.sepia.ceph.com 2024-04-23T01:18:05.639 DEBUG:teuthology.orchestra.run.smithi121:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-23T01:18:05.662 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi153.front.sepia.ceph.com 2024-04-23T01:18:05.663 DEBUG:teuthology.orchestra.run.smithi153:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-23T01:18:05.687 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-23T01:18:05.687 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-23T01:18:05.704 DEBUG:teuthology.orchestra.run.smithi153:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-23T01:18:05.788 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-23T01:18:05.788 DEBUG:teuthology.orchestra.run.smithi121:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-23T01:18:05.790 DEBUG:teuthology.orchestra.run.smithi153:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-23T01:18:05.941 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-23T01:18:05.953 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-23T01:18:05.954 DEBUG:teuthology.orchestra.run.smithi121:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-23T01:18:05.981 DEBUG:teuthology.orchestra.run.smithi153:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-23T01:18:06.009 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-23T01:18:06.019 DEBUG:teuthology.orchestra.run.smithi121:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-23T01:18:06.023 DEBUG:teuthology.orchestra.run.smithi153:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-23T01:18:06.047 INFO:teuthology.orchestra.run.smithi121.stdout:kernel.core_pattern = core 2024-04-23T01:18:06.075 INFO:teuthology.orchestra.run.smithi153.stdout:kernel.core_pattern = core 2024-04-23T01:18:06.105 DEBUG:teuthology.orchestra.run.smithi121:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-23T01:18:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T01:18:06.134 DEBUG:teuthology.orchestra.run.smithi153:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-23T01:18:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-23T01:18:06.163 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-23T01:18:06.175 INFO:teuthology.task.internal:Transferring archived files... 2024-04-23T01:18:06.176 DEBUG:teuthology.misc:Transferring archived files from smithi121:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi121 2024-04-23T01:18:06.177 DEBUG:teuthology.orchestra.run.smithi121:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-23T01:18:06.224 DEBUG:teuthology.misc:Transferring archived files from smithi153:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_22:08:02-orch-reef-distro-default-smithi/7668995/remote/smithi153 2024-04-23T01:18:06.224 DEBUG:teuthology.orchestra.run.smithi153:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-23T01:18:06.267 INFO:teuthology.task.internal:Removing archive directory... 2024-04-23T01:18:06.267 DEBUG:teuthology.orchestra.run.smithi121:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-23T01:18:06.269 DEBUG:teuthology.orchestra.run.smithi153:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-23T01:18:06.317 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-23T01:18:06.329 INFO:teuthology.task.internal:Not uploading archives. 2024-04-23T01:18:06.329 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-23T01:18:06.390 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-23T01:18:06.391 DEBUG:teuthology.orchestra.run.smithi121:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-23T01:18:06.393 DEBUG:teuthology.orchestra.run.smithi153:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-23T01:18:06.408 INFO:teuthology.orchestra.run.smithi121.stdout: 525357 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 23 01:18 /home/ubuntu/cephtest 2024-04-23T01:18:06.410 INFO:teuthology.orchestra.run.smithi153.stdout: 525364 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 23 01:18 /home/ubuntu/cephtest 2024-04-23T01:18:06.412 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-23T01:18:06.429 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-23T01:18:06.499 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/16.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2660.874051809311 owner: scheduled_teuthology@teuthology success: true 2024-04-23T01:18:06.499 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-23T01:18:06.596 INFO:teuthology.run:pass