2024-03-11T20:18:13.826 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-11T20:18:13.826 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732 --owner scheduled_teuthology@teuthology --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose -- /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/orig.config.yaml 2024-03-11T20:18:13.865 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-11T20:18:13.948 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732 branch: main description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7591732' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 2b0101eb74d60610d544cb8a473e23a35ac1cc0b ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 2b0101eb74d60610d544cb8a473e23a35ac1cc0b selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: main sha1: 1c349451176cc5b4ebfb24b22eaaa754e05cff6c owner: scheduled_teuthology@teuthology priority: 200 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6815 sha1: 2b0101eb74d60610d544cb8a473e23a35ac1cc0b sleep_before_teardown: 0 subset: 10/64 suite: orch suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c349451176cc5b4ebfb24b22eaaa754e05cff6c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c349451176cc5b4ebfb24b22eaaa754e05cff6c targets: smithi083.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDJovG9nEoNTNJG3ojgLf5glFR6waL5+aaAGl1CRgUJ30B8vT/HvrH8awZZ4XqoY4Dl8fYEutRwuFjfaV+cttFk= smithi204.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCXcG/ZdMH6kFIx8fh2ZW0FgkKbnzw3np2nZJt4qUtWpCV68lGgo2UMuTWTXlwB/rL2Az3njM2gLjxLDpvyQWJw= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-11_20:08:13 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-03-11T20:18:13.949 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c349451176cc5b4ebfb24b22eaaa754e05cff6c/qa; will attempt to use it 2024-03-11T20:18:13.950 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c349451176cc5b4ebfb24b22eaaa754e05cff6c/qa/tasks 2024-03-11T20:18:13.950 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-11T20:18:13.951 INFO:teuthology.task.internal:Checking packages... 2024-03-11T20:18:13.978 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '2b0101eb74d60610d544cb8a473e23a35ac1cc0b' 2024-03-11T20:18:13.978 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-11T20:18:13.978 INFO:teuthology.packaging:ref: None 2024-03-11T20:18:13.979 INFO:teuthology.packaging:tag: None 2024-03-11T20:18:13.979 INFO:teuthology.packaging:branch: main 2024-03-11T20:18:13.979 INFO:teuthology.packaging:sha1: 2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:18:13.979 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-03-11T20:18:14.496 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1979.g2b0101eb 2024-03-11T20:18:14.498 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-11T20:18:14.504 INFO:teuthology.task.internal:no buildpackages task found 2024-03-11T20:18:14.504 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-11T20:18:14.521 INFO:teuthology.task.internal:Saving configuration 2024-03-11T20:18:14.536 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-11T20:18:14.542 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-11T20:18:14.567 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-11 20:13:13.635943', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDJovG9nEoNTNJG3ojgLf5glFR6waL5+aaAGl1CRgUJ30B8vT/HvrH8awZZ4XqoY4Dl8fYEutRwuFjfaV+cttFk='} 2024-03-11T20:18:14.589 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi204.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-11 20:13:13.637506', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCXcG/ZdMH6kFIx8fh2ZW0FgkKbnzw3np2nZJt4qUtWpCV68lGgo2UMuTWTXlwB/rL2Az3njM2gLjxLDpvyQWJw='} 2024-03-11T20:18:14.589 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-11T20:18:14.595 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-03-11T20:18:14.595 INFO:teuthology.task.internal:roles: ubuntu@smithi204.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-03-11T20:18:14.595 INFO:teuthology.run_tasks:Running task console_log... 2024-03-11T20:18:14.716 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fda10a5ef70>, signals=[15]) 2024-03-11T20:18:14.719 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-11T20:18:14.740 INFO:teuthology.task.internal:Opening connections... 2024-03-11T20:18:14.741 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:18:14.744 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:18:14.862 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi204.front.sepia.ceph.com 2024-03-11T20:18:14.863 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:18:14.965 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-11T20:18:14.974 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2024-03-11T20:18:14.994 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2024-03-11T20:18:14.996 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2024-03-11T20:18:15.053 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2024-03-11T20:18:15.053 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="9" 2024-03-11T20:18:15.053 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2024-03-11T20:18:15.053 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="9" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el9" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:LOGO="fedora-logo-icon" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-11T20:18:15.054 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2024-03-11T20:18:15.055 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-11T20:18:15.055 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-11T20:18:15.055 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-11T20:18:15.056 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2024-03-11T20:18:15.080 DEBUG:teuthology.orchestra.run.smithi204:> uname -m 2024-03-11T20:18:15.097 INFO:teuthology.orchestra.run.smithi204.stdout:x86_64 2024-03-11T20:18:15.097 DEBUG:teuthology.orchestra.run.smithi204:> cat /etc/os-release 2024-03-11T20:18:15.155 INFO:teuthology.orchestra.run.smithi204.stdout:NAME="CentOS Stream" 2024-03-11T20:18:15.155 INFO:teuthology.orchestra.run.smithi204.stdout:VERSION="9" 2024-03-11T20:18:15.155 INFO:teuthology.orchestra.run.smithi204.stdout:ID="centos" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:ID_LIKE="rhel fedora" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:VERSION_ID="9" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:PLATFORM_ID="platform:el9" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:ANSI_COLOR="0;31" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:LOGO="fedora-logo-icon" 2024-03-11T20:18:15.156 INFO:teuthology.orchestra.run.smithi204.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-11T20:18:15.157 INFO:teuthology.orchestra.run.smithi204.stdout:HOME_URL="https://centos.org/" 2024-03-11T20:18:15.157 INFO:teuthology.orchestra.run.smithi204.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-11T20:18:15.157 INFO:teuthology.orchestra.run.smithi204.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-11T20:18:15.157 INFO:teuthology.orchestra.run.smithi204.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-11T20:18:15.157 INFO:teuthology.lock.ops:Updating smithi204.front.sepia.ceph.com on lock server 2024-03-11T20:18:15.171 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-11T20:18:15.184 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-11T20:18:15.200 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-11T20:18:15.200 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2024-03-11T20:18:15.202 DEBUG:teuthology.orchestra.run.smithi204:> test '!' -e /home/ubuntu/cephtest 2024-03-11T20:18:15.219 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-11T20:18:15.225 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-11T20:18:15.225 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2024-03-11T20:18:15.259 DEBUG:teuthology.orchestra.run.smithi204:> test -z $(ls -A /var/lib/ceph) 2024-03-11T20:18:15.331 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-11T20:18:15.389 INFO:teuthology.run_tasks:Running task kernel... 2024-03-11T20:18:15.405 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-11T20:18:15.405 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-11T20:18:15.405 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-11T20:18:15.406 DEBUG:teuthology.orchestra.run.smithi204:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-11T20:18:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:18:15.422 DEBUG:teuthology.orchestra.run.smithi204:> uname -r 2024-03-11T20:18:15.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:18:15.423 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-03-11T20:18:15.478 INFO:teuthology.orchestra.run.smithi204.stdout:5.14.0-361.el9.x86_64 2024-03-11T20:18:15.478 INFO:teuthology.task.kernel:Running kernel on smithi204: 5.14.0-361.el9.x86_64 2024-03-11T20:18:15.478 DEBUG:teuthology.orchestra.run.smithi204:> sudo yum install -y kernel 2024-03-11T20:18:15.480 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-361.el9.x86_64 2024-03-11T20:18:15.480 INFO:teuthology.task.kernel:Running kernel on smithi083: 5.14.0-361.el9.x86_64 2024-03-11T20:18:15.481 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2024-03-11T20:18:17.620 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS-9 - AppStream 12 MB/s | 6.7 MB 00:00 2024-03-11T20:18:17.649 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-11T20:18:19.029 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS-9 - BaseOS 6.1 MB/s | 2.0 MB 00:00 2024-03-11T20:18:19.391 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - BaseOS 3.1 MB/s | 2.0 MB 00:00 2024-03-11T20:18:23.087 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 2.4 MB/s | 8.0 MB 00:03 2024-03-11T20:18:24.827 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - BaseOS 1.4 MB/s | 8.0 MB 00:05 2024-03-11T20:18:25.833 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-03-11T20:18:26.736 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - AppStream 22 MB/s | 19 MB 00:00 2024-03-11T20:18:30.833 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.1 MB 00:01 2024-03-11T20:18:30.862 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.1 MB 00:00 2024-03-11T20:18:32.392 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 15 kB 00:00 2024-03-11T20:18:32.471 INFO:teuthology.orchestra.run.smithi204.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 15 kB 00:00 2024-03-11T20:18:36.118 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 5.7 MB/s | 21 MB 00:03 2024-03-11T20:18:41.052 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-03-11T20:18:43.217 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-11T20:18:43.217 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-11T20:18:43.316 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-03-11T20:18:43.326 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-03-11T20:18:43.326 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-03-11T20:18:43.326 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-03-11T20:18:43.326 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-03-11T20:18:43.326 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 5.14.0-427.el9 baseos 6.0 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core x86_64 5.14.0-427.el9 baseos 21 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules x86_64 5.14.0-427.el9 baseos 39 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core x86_64 5.14.0-427.el9 baseos 33 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 100 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 123 M 2024-03-11T20:18:43.327 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-03-11T20:18:44.170 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): kernel-5.14.0-427.el9.x86_64.rpm 12 MB/s | 6.0 MB 00:00 2024-03-11T20:18:44.937 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): kernel-core-5.14.0-427.el9.x86_64.rpm 16 MB/s | 21 MB 00:01 2024-03-11T20:18:45.803 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): kernel-modules-5.14.0-427.el9.x86_64.rpm 18 MB/s | 39 MB 00:02 2024-03-11T20:18:46.287 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): kernel-modules-core-5.14.0-427.el9.x86_6 16 MB/s | 33 MB 00:02 2024-03-11T20:18:46.287 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-03-11T20:18:46.287 INFO:teuthology.orchestra.run.smithi083.stdout:Total 34 MB/s | 100 MB 00:02 2024-03-11T20:18:47.172 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-03-11T20:18:47.535 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-03-11T20:18:47.535 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-03-11T20:18:48.865 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-03-11T20:18:48.866 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-03-11T20:18:51.593 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-03-11T20:18:52.228 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-core-5.14.0-427.el9.x86_64 1/4 2024-03-11T20:18:52.364 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:18:55.177 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:18:55.377 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:19:00.129 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:19:00.262 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:05.044 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:34.182 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:34.182 INFO:teuthology.orchestra.run.smithi083.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-11T20:19:34.182 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:19:34.268 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:34.700 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:34.700 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-5.14.0-427.el9.x86_64 1/4 2024-03-11T20:19:34.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:19:34.701 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.105 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.106 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:19:35.106 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-03-11T20:19:35.612 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-03-11T20:19:35.770 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.770 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-11T20:19:35.771 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-427.el9.x86_64 2024-03-11T20:19:35.771 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-11T20:19:35.771 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-11T20:19:35.771 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-03-11T20:19:35.771 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi083.front.sepia.ceph.com, path=None, version=distro) 2024-03-11T20:19:35.771 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-03-11T20:19:35.891 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-11T20:19:35.892 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-11T20:19:35.892 DEBUG:teuthology.orchestra.run.smithi083:> sudo rpm -qi grub2-tools 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Name : grub2-tools 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Epoch : 1 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Version : 2.06 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Release : 68.el9 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Architecture: x86_64 2024-03-11T20:19:35.936 INFO:teuthology.orchestra.run.smithi083.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Group : Unspecified 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Size : 8264254 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:License : GPLv3+ 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-11T20:19:35.937 INFO:teuthology.orchestra.run.smithi083.stdout:Packager : builder@centos.org 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:Vendor : CentOS 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:Summary : Support tools for GRUB. 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:Description : 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-11T20:19:35.938 INFO:teuthology.orchestra.run.smithi083.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-11T20:19:35.939 INFO:teuthology.orchestra.run.smithi083.stdout:hardware devices. 2024-03-11T20:19:35.939 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:19:35.939 INFO:teuthology.orchestra.run.smithi083.stdout:This subpackage provides tools for support of all platforms. 2024-03-11T20:19:35.940 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-11T20:19:35.940 INFO:teuthology.task.kernel:Updating grub on smithi083 to boot 5.14.0-427.el9.x86_64 2024-03-11T20:19:35.940 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-11T20:19:36.588 INFO:teuthology.orchestra.run.smithi083.stderr:Generating grub configuration file ... 2024-03-11T20:19:37.952 INFO:teuthology.orchestra.run.smithi083.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-11T20:19:37.995 INFO:teuthology.orchestra.run.smithi083.stderr:done 2024-03-11T20:19:37.997 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-03-11T20:19:38.012 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.LOtIBoPvsl 2024-03-11T20:19:38.013 DEBUG:teuthology.orchestra.run.smithi083:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.LOtIBoPvsl 2024-03-11T20:19:38.080 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.LOtIBoPvsl 2024-03-11T20:19:38.219 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.LOtIBoPvsl is 6KB 2024-03-11T20:19:38.267 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.LOtIBoPvsl 2024-03-11T20:19:38.281 DEBUG:teuthology.orchestra.run.smithi083:> sudo /bin/ls /boot/loader/entries || true 2024-03-11T20:19:38.346 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-11T20:19:38.346 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-11T20:19:38.346 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-11T20:19:38.346 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64.conf 2024-03-11T20:19:38.347 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64 2024-03-11T20:19:38.503 DEBUG:teuthology.orchestra.run.smithi083:> sudo shutdown -r now 2024-03-11T20:20:08.506 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-03-11T20:20:08.507 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:20:08.507 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:20:18.569 INFO:teuthology.orchestra.run.smithi204.stdout:Extra Packages for Enterprise Linux 199 kB/s | 21 MB 01:45 2024-03-11T20:20:23.569 INFO:teuthology.orchestra.run.smithi204.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-03-11T20:20:25.768 INFO:teuthology.orchestra.run.smithi204.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-11T20:20:25.768 INFO:teuthology.orchestra.run.smithi204.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-11T20:20:25.854 INFO:teuthology.orchestra.run.smithi204.stdout:Dependencies resolved. 2024-03-11T20:20:25.863 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-03-11T20:20:25.863 INFO:teuthology.orchestra.run.smithi204.stdout: Package Architecture Version Repository Size 2024-03-11T20:20:25.863 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout:Installing: 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: kernel x86_64 5.14.0-427.el9 baseos 6.0 M 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout:Installing dependencies: 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-core x86_64 5.14.0-427.el9 baseos 21 M 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules x86_64 5.14.0-427.el9 baseos 39 M 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-core x86_64 5.14.0-427.el9 baseos 33 M 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction Summary 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout:================================================================================ 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout:Install 4 Packages 2024-03-11T20:20:25.864 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:20:25.865 INFO:teuthology.orchestra.run.smithi204.stdout:Total download size: 100 M 2024-03-11T20:20:25.865 INFO:teuthology.orchestra.run.smithi204.stdout:Installed size: 123 M 2024-03-11T20:20:25.865 INFO:teuthology.orchestra.run.smithi204.stdout:Downloading Packages: 2024-03-11T20:20:26.554 INFO:teuthology.orchestra.run.smithi204.stdout:(1/4): kernel-5.14.0-427.el9.x86_64.rpm 9.3 MB/s | 6.0 MB 00:00 2024-03-11T20:20:27.038 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-03-11T20:20:27.413 INFO:teuthology.orchestra.run.smithi204.stdout:(2/4): kernel-core-5.14.0-427.el9.x86_64.rpm 14 MB/s | 21 MB 00:01 2024-03-11T20:20:28.371 INFO:teuthology.orchestra.run.smithi204.stdout:(3/4): kernel-modules-5.14.0-427.el9.x86_64.rpm 16 MB/s | 39 MB 00:02 2024-03-11T20:20:28.738 INFO:teuthology.orchestra.run.smithi204.stdout:(4/4): kernel-modules-core-5.14.0-427.el9.x86_6 15 MB/s | 33 MB 00:02 2024-03-11T20:20:28.738 INFO:teuthology.orchestra.run.smithi204.stdout:-------------------------------------------------------------------------------- 2024-03-11T20:20:28.738 INFO:teuthology.orchestra.run.smithi204.stdout:Total 35 MB/s | 100 MB 00:02 2024-03-11T20:20:29.622 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction check 2024-03-11T20:20:30.006 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction check succeeded. 2024-03-11T20:20:30.006 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction test 2024-03-11T20:20:31.360 INFO:teuthology.orchestra.run.smithi204.stdout:Transaction test succeeded. 2024-03-11T20:20:31.361 INFO:teuthology.orchestra.run.smithi204.stdout:Running transaction 2024-03-11T20:20:34.306 INFO:teuthology.orchestra.run.smithi204.stdout: Preparing : 1/1 2024-03-11T20:20:34.989 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-modules-core-5.14.0-427.el9.x86_64 1/4 2024-03-11T20:20:35.123 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:20:36.039 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:20:36.040 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:20:37.949 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:20:38.136 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:20:42.896 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:20:43.088 INFO:teuthology.orchestra.run.smithi204.stdout: Installing : kernel-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:20:47.857 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:21:17.391 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:21:17.392 INFO:teuthology.orchestra.run.smithi204.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-11T20:21:17.392 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:21:17.477 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-modules-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:21:17.902 INFO:teuthology.orchestra.run.smithi204.stdout: Running scriptlet: kernel-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:21:17.903 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-5.14.0-427.el9.x86_64 1/4 2024-03-11T20:21:17.903 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-core-5.14.0-427.el9.x86_64 2/4 2024-03-11T20:21:17.903 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-modules-5.14.0-427.el9.x86_64 3/4 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: Verifying : kernel-modules-core-5.14.0-427.el9.x86_64 4/4 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout:Installed: 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-5.14.0-427.el9.x86_64 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-core-5.14.0-427.el9.x86_64 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-5.14.0-427.el9.x86_64 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: kernel-modules-core-5.14.0-427.el9.x86_64 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:21:18.337 INFO:teuthology.orchestra.run.smithi204.stdout:Complete! 2024-03-11T20:21:18.868 DEBUG:teuthology.orchestra.run.smithi204:> rpm -q kernel | sort -rV | head -n 1 2024-03-11T20:21:19.021 INFO:teuthology.orchestra.run.smithi204.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-11T20:21:19.021 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-11T20:21:19.021 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-427.el9.x86_64 2024-03-11T20:21:19.022 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-11T20:21:19.022 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-11T20:21:19.022 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-03-11T20:21:19.022 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi204.front.sepia.ceph.com, path=None, version=distro) 2024-03-11T20:21:19.022 DEBUG:teuthology.orchestra.run.smithi204:> rpm -q kernel | sort -rV | head -n 1 2024-03-11T20:21:19.147 INFO:teuthology.orchestra.run.smithi204.stdout:kernel-5.14.0-427.el9.x86_64 2024-03-11T20:21:19.147 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-427.el9.x86_64 2024-03-11T20:21:19.147 DEBUG:teuthology.orchestra.run.smithi204:> sudo rpm -qi grub2-tools 2024-03-11T20:21:19.190 INFO:teuthology.orchestra.run.smithi204.stdout:Name : grub2-tools 2024-03-11T20:21:19.190 INFO:teuthology.orchestra.run.smithi204.stdout:Epoch : 1 2024-03-11T20:21:19.190 INFO:teuthology.orchestra.run.smithi204.stdout:Version : 2.06 2024-03-11T20:21:19.190 INFO:teuthology.orchestra.run.smithi204.stdout:Release : 68.el9 2024-03-11T20:21:19.190 INFO:teuthology.orchestra.run.smithi204.stdout:Architecture: x86_64 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Group : Unspecified 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Size : 8264254 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:License : GPLv3+ 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Packager : builder@centos.org 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Vendor : CentOS 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Summary : Support tools for GRUB. 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout:Description : 2024-03-11T20:21:19.191 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout:hardware devices. 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:21:19.192 INFO:teuthology.orchestra.run.smithi204.stdout:This subpackage provides tools for support of all platforms. 2024-03-11T20:21:19.193 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-11T20:21:19.193 INFO:teuthology.task.kernel:Updating grub on smithi204 to boot 5.14.0-427.el9.x86_64 2024-03-11T20:21:19.193 DEBUG:teuthology.orchestra.run.smithi204:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-11T20:21:19.821 INFO:teuthology.orchestra.run.smithi204.stderr:Generating grub configuration file ... 2024-03-11T20:21:21.182 INFO:teuthology.orchestra.run.smithi204.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-11T20:21:21.222 INFO:teuthology.orchestra.run.smithi204.stderr:done 2024-03-11T20:21:21.224 DEBUG:teuthology.orchestra.run.smithi204:> mktemp 2024-03-11T20:21:21.239 INFO:teuthology.orchestra.run.smithi204.stdout:/tmp/tmp.5I3ZtgNHZh 2024-03-11T20:21:21.239 DEBUG:teuthology.orchestra.run.smithi204:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.5I3ZtgNHZh 2024-03-11T20:21:21.308 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod 0666 /tmp/tmp.5I3ZtgNHZh 2024-03-11T20:21:21.451 DEBUG:teuthology.orchestra.remote:smithi204:/tmp/tmp.5I3ZtgNHZh is 6KB 2024-03-11T20:21:21.472 DEBUG:teuthology.orchestra.run.smithi204:> rm -fr /tmp/tmp.5I3ZtgNHZh 2024-03-11T20:21:21.487 DEBUG:teuthology.orchestra.run.smithi204:> sudo /bin/ls /boot/loader/entries || true 2024-03-11T20:21:21.555 INFO:teuthology.orchestra.run.smithi204.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-11T20:21:21.556 INFO:teuthology.orchestra.run.smithi204.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-11T20:21:21.556 INFO:teuthology.orchestra.run.smithi204.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-11T20:21:21.556 INFO:teuthology.orchestra.run.smithi204.stdout:15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64.conf 2024-03-11T20:21:21.557 DEBUG:teuthology.orchestra.run.smithi204:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-427.el9.x86_64 2024-03-11T20:21:21.695 DEBUG:teuthology.orchestra.run.smithi204:> sudo shutdown -r now 2024-03-11T20:21:36.047 DEBUG:teuthology.orchestra.remote:timed out 2024-03-11T20:21:48.058 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:21:48.060 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:21:48.343 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-03-11T20:21:48.710 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:21:48.710 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-427.el9.x86_64"... 2024-03-11T20:21:48.710 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-03-11T20:21:48.725 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-427.el9.x86_64 2024-03-11T20:21:48.725 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-427.el9.x86_64 vs 5.14.0-427.el9.x86_64 2024-03-11T20:21:48.725 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-11T20:21:48.725 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-11T20:21:49.726 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-03-11T20:21:49.806 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-11T20:21:50.131 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2024-03-11T20:21:50.163 DEBUG:teuthology.parallel:result is None 2024-03-11T20:21:51.698 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-03-11T20:21:51.698 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:21:51.699 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:22:26.340 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.204 2024-03-11T20:22:35.341 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:22:35.342 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:22:53.726 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.204 2024-03-11T20:23:05.727 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:23:05.728 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:23:06.169 DEBUG:teuthology.orchestra.run.smithi204:> true 2024-03-11T20:23:06.666 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:23:06.666 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-427.el9.x86_64"... 2024-03-11T20:23:06.667 DEBUG:teuthology.orchestra.run.smithi204:> uname -r 2024-03-11T20:23:06.680 INFO:teuthology.orchestra.run.smithi204.stdout:5.14.0-427.el9.x86_64 2024-03-11T20:23:06.681 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-427.el9.x86_64 vs 5.14.0-427.el9.x86_64 2024-03-11T20:23:06.681 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-11T20:23:06.681 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-11T20:23:07.682 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-03-11T20:23:07.682 DEBUG:teuthology.orchestra.run.smithi204:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-11T20:23:07.775 INFO:teuthology.orchestra.run.smithi204.stdout:ttyS1 2024-03-11T20:23:07.812 DEBUG:teuthology.parallel:result is None 2024-03-11T20:23:07.812 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-11T20:23:07.890 INFO:teuthology.task.internal:Creating test directory... 2024-03-11T20:23:07.891 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-11T20:23:07.896 DEBUG:teuthology.orchestra.run.smithi204:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-11T20:23:07.912 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-11T20:23:07.931 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-11T20:23:08.028 INFO:teuthology.task.internal:Creating archive directory... 2024-03-11T20:23:08.028 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-11T20:23:08.032 DEBUG:teuthology.orchestra.run.smithi204:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-11T20:23:08.246 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-11T20:23:08.252 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-11T20:23:08.253 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-11T20:23:08.258 DEBUG:teuthology.orchestra.run.smithi204:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-11T20:23:08.288 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-11T20:23:08.290 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-11T20:23:08.302 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-11T20:23:08.303 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-11T20:23:08.304 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-11T20:23:08.311 INFO:teuthology.task.internal:Configuring sudo... 2024-03-11T20:23:08.312 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-11T20:23:08.347 DEBUG:teuthology.orchestra.run.smithi204:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-11T20:23:08.375 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-11T20:23:08.384 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-11T20:23:08.385 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-11T20:23:08.416 DEBUG:teuthology.orchestra.run.smithi204:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-11T20:23:08.431 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-11T20:23:08.515 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-11T20:23:08.610 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-11T20:23:08.653 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-11T20:23:08.696 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:23:08.696 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-11T20:23:08.763 DEBUG:teuthology.orchestra.run.smithi204:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-11T20:23:08.802 DEBUG:teuthology.orchestra.run.smithi204:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-11T20:23:08.858 DEBUG:teuthology.orchestra.run.smithi204:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-11T20:23:08.941 DEBUG:teuthology.orchestra.run.smithi204:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-11T20:23:08.985 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:23:08.985 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-11T20:23:09.054 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2024-03-11T20:23:09.056 DEBUG:teuthology.orchestra.run.smithi204:> sudo service rsyslog restart 2024-03-11T20:23:09.104 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-11T20:23:09.141 INFO:teuthology.orchestra.run.smithi204.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-11T20:23:09.583 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-11T20:23:09.591 INFO:teuthology.task.internal:Starting timer... 2024-03-11T20:23:09.591 INFO:teuthology.run_tasks:Running task pcp... 2024-03-11T20:23:09.602 INFO:teuthology.run_tasks:Running task selinux... 2024-03-11T20:23:09.611 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-03-11T20:23:09.611 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2024-03-11T20:23:09.671 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2024-03-11T20:23:09.673 DEBUG:teuthology.orchestra.run.smithi204:> sudo service auditd rotate 2024-03-11T20:23:09.730 INFO:teuthology.orchestra.run.smithi204.stdout:Rotating logs: 2024-03-11T20:23:09.732 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-11T20:23:09.732 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2024-03-11T20:23:09.754 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2024-03-11T20:23:09.754 DEBUG:teuthology.orchestra.run.smithi204:> /usr/sbin/getenforce 2024-03-11T20:23:09.794 INFO:teuthology.orchestra.run.smithi204.stdout:Permissive 2024-03-11T20:23:09.794 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi204.front.sepia.ceph.com': 'permissive'} 2024-03-11T20:23:09.794 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-11T20:23:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:23:09.823 DEBUG:teuthology.orchestra.run.smithi204:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-11T20:23:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:23:09.862 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-11T20:23:09.862 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2024-03-11T20:23:09.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo /usr/sbin/setenforce permissive 2024-03-11T20:23:09.937 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-11T20:23:09.947 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-11T20:23:09.964 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-11T20:23:09.965 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com,smithi204.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-11T20:27:21.678 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com'), Remote(name='ubuntu@smithi204.front.sepia.ceph.com')] 2024-03-11T20:27:21.679 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:27:21.680 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:27:21.771 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-03-11T20:27:21.849 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-03-11T20:27:21.850 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:27:21.850 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi204.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-11T20:27:21.937 DEBUG:teuthology.orchestra.run.smithi204:> true 2024-03-11T20:27:22.016 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi204.front.sepia.ceph.com' 2024-03-11T20:27:22.017 INFO:teuthology.run_tasks:Running task clock... 2024-03-11T20:27:22.027 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-11T20:27:22.027 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-11T20:27:22.028 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-11T20:27:22.030 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-11T20:27:22.030 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-11T20:27:22.058 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-11T20:27:22.074 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-11T20:27:22.087 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-11T20:27:22.103 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2024-03-11T20:27:22.104 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-11T20:27:22.117 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2024-03-11T20:27:22.134 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-11T20:27:22.134 INFO:teuthology.orchestra.run.smithi204.stderr:sudo: ntpd: command not found 2024-03-11T20:27:22.146 INFO:teuthology.orchestra.run.smithi204.stdout:506 Cannot talk to daemon 2024-03-11T20:27:22.150 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-11T20:27:22.163 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-11T20:27:22.179 INFO:teuthology.orchestra.run.smithi204.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-11T20:27:22.208 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-03-11T20:27:22.211 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-11T20:27:22.212 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-03-11T20:27:22.212 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.212 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.212 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.212 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.239 INFO:teuthology.orchestra.run.smithi204.stderr:bash: line 1: ntpq: command not found 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:=============================================================================== 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.243 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:27:22.243 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-11T20:27:22.339 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '2b0101eb74d60610d544cb8a473e23a35ac1cc0b'} 2024-03-11T20:27:22.339 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-03-11T20:27:22.340 INFO:tasks.cephadm:Cluster fsid is c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:27:22.340 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-11T20:27:22.340 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-03-11T20:27:22.340 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi083': '172.21.15.83', 'mon.smithi204': '172.21.15.204'} 2024-03-11T20:27:22.340 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-11T20:27:22.340 DEBUG:teuthology.orchestra.run.smithi083:> sudo hostname $(hostname -s) 2024-03-11T20:27:22.369 DEBUG:teuthology.orchestra.run.smithi204:> sudo hostname $(hostname -s) 2024-03-11T20:27:22.397 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-03-11T20:27:22.397 DEBUG:teuthology.orchestra.run.smithi083:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.022 INFO:teuthology.orchestra.run.smithi083.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 11 20:27 /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.022 DEBUG:teuthology.orchestra.run.smithi204:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.597 INFO:teuthology.orchestra.run.smithi204.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 11 20:27 /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.597 DEBUG:teuthology.orchestra.run.smithi083:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.616 DEBUG:teuthology.orchestra.run.smithi204:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-11T20:27:23.644 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-03-11T20:27:23.644 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-11T20:27:23.658 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-11T20:27:24.529 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-11T20:27:24.552 INFO:teuthology.orchestra.run.smithi204.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-11T20:27:46.132 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "repo_digests": [ 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-03-11T20:27:46.133 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:27:58.345 INFO:teuthology.orchestra.run.smithi204.stdout:{ 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "repo_digests": [ 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout: ] 2024-03-11T20:27:58.346 INFO:teuthology.orchestra.run.smithi204.stdout:} 2024-03-11T20:27:58.363 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph 2024-03-11T20:27:58.399 DEBUG:teuthology.orchestra.run.smithi204:> sudo mkdir -p /etc/ceph 2024-03-11T20:27:58.434 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /etc/ceph 2024-03-11T20:27:58.469 DEBUG:teuthology.orchestra.run.smithi204:> sudo chmod 777 /etc/ceph 2024-03-11T20:27:58.501 INFO:tasks.cephadm:Writing seed config... 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-11T20:27:58.502 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-11T20:27:58.503 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-11T20:27:58.503 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:27:58.503 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-11T20:27:58.525 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = c3f853a8-dfe5-11ee-95c8-87774f69a715 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-11T20:27:58.526 DEBUG:teuthology.orchestra.run.smithi083:mon.smithi083> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service 2024-03-11T20:27:58.568 INFO:tasks.cephadm:Bootstrapping... 2024-03-11T20:27:58.569 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.83 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:27:58.767 INFO:teuthology.orchestra.run.smithi083.stderr:-------------------------------------------------------------------------------- 2024-03-11T20:27:58.768 INFO:teuthology.orchestra.run.smithi083.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'c3f853a8-dfe5-11ee-95c8-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.83', '--skip-admin-label'] 2024-03-11T20:27:58.791 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.3-dev 2024-03-11T20:27:58.793 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying podman|docker is present... 2024-03-11T20:27:58.814 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.3-dev 2024-03-11T20:27:58.816 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying lvm2 is present... 2024-03-11T20:27:58.816 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying time synchronization is in place... 2024-03-11T20:27:58.823 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-11T20:27:58.829 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:27:58.836 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-03-11T20:27:58.841 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-03-11T20:27:58.842 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-03-11T20:27:58.842 INFO:teuthology.orchestra.run.smithi083.stderr:Repeating the final host check... 2024-03-11T20:27:58.862 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.3-dev 2024-03-11T20:27:58.864 INFO:teuthology.orchestra.run.smithi083.stderr:podman (/bin/podman) version 4.9.3 is present 2024-03-11T20:27:58.864 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl is present 2024-03-11T20:27:58.865 INFO:teuthology.orchestra.run.smithi083.stderr:lvcreate is present 2024-03-11T20:27:58.870 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-11T20:27:58.876 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:27:58.883 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-03-11T20:27:58.888 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Host looks OK 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Cluster fsid: c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Acquiring lock 140459680895328 on /run/cephadm/c3f853a8-dfe5-11ee-95c8-87774f69a715.lock 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Lock 140459680895328 acquired on /run/cephadm/c3f853a8-dfe5-11ee-95c8-87774f69a715.lock 2024-03-11T20:27:58.890 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 3300 ... 2024-03-11T20:27:58.891 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 6789 ... 2024-03-11T20:27:58.891 INFO:teuthology.orchestra.run.smithi083.stderr:Base mon IP is 172.21.15.83, final addrv is [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-03-11T20:27:58.892 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.83 metric 100 2024-03-11T20:27:58.893 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.83 metric 100 2024-03-11T20:27:58.895 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-03-11T20:27:58.895 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-11T20:27:58.895 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1797sec hoplimit 64 pref medium 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:ce91/64 scope link 2024-03-11T20:27:58.898 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-11T20:27:58.899 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2024-03-11T20:27:58.899 INFO:teuthology.orchestra.run.smithi083.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-11T20:27:58.899 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-11T20:27:58.999 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Getting image source signatures 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-03-11T20:27:59.770 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-03-11T20:27:59.771 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-11T20:27:59.911 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Writing manifest to image destination 2024-03-11T20:28:00.063 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-11T20:28:00.569 INFO:teuthology.orchestra.run.smithi083.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-11T20:28:00.996 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-11T20:28:00.996 INFO:teuthology.orchestra.run.smithi083.stderr:Extracting ceph user uid/gid from container image... 2024-03-11T20:28:01.379 INFO:teuthology.orchestra.run.smithi083.stderr:stat: 167 167 2024-03-11T20:28:01.927 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial keys... 2024-03-11T20:28:02.285 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQBSae9lDrjmDBAA+SiLbtmNWIRCzO0G2QxxGA== 2024-03-11T20:28:03.149 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQBTae9laRFWBRAAkVsi8Xe0Iwp0Df3DU6z7IQ== 2024-03-11T20:28:03.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQBTae9lX8rVNhAAkhnbGwaXjVoKYV/Q+Op++Q== 2024-03-11T20:28:04.428 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial monmap... 2024-03-11T20:28:04.785 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-11T20:28:04.785 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-03-11T20:28:04.785 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:04.785 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-11T20:28:05.514 INFO:teuthology.orchestra.run.smithi083.stderr:monmaptool for smithi083 [v2:172.21.15.83:3300,v1:172.21.15.83:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-11T20:28:05.514 INFO:teuthology.orchestra.run.smithi083.stderr:setting min_mon_release = octopus 2024-03-11T20:28:05.514 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: set fsid to c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:05.514 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-11T20:28:05.514 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:28:05.515 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mon... 2024-03-11T20:28:15.430 INFO:teuthology.orchestra.run.smithi083.stderr:create mon.smithi083 on 2024-03-11T20:28:15.720 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-11T20:28:15.972 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target → /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target. 2024-03-11T20:28:15.972 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target → /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target. 2024-03-11T20:28:16.571 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service: Unit ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service not loaded. 2024-03-11T20:28:16.578 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target.wants/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service → /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@.service. 2024-03-11T20:28:17.289 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-03-11T20:28:17.296 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:28:17.297 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-03-11T20:28:17.297 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-03-11T20:28:17.298 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon to start... 2024-03-11T20:28:17.298 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon... 2024-03-11T20:28:17.411 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:17 smithi083 systemd[1]: Started Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:28:17.811 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: cluster: 2024-03-11T20:28:17.811 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: id: c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:17.811 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-03-11T20:28:17.811 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:17.811 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: services: 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi083 (age 0.596794s) 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr: no daemons active 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: data: 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pgs: 2024-03-11T20:28:17.812 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:18.319 INFO:teuthology.orchestra.run.smithi083.stderr:mon is available 2024-03-11T20:28:18.319 INFO:teuthology.orchestra.run.smithi083.stderr:Assimilating anything we can from ceph.conf... 2024-03-11T20:28:18.425 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: mon.smithi083 is new leader, mons smithi083 in quorum (ranks 0) 2024-03-11T20:28:18.426 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: monmap e1: 1 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-03-11T20:28:18.426 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: fsmap 2024-03-11T20:28:18.426 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: osdmap e1: 0 total, 0 up, 0 in 2024-03-11T20:28:18.426 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: mgrmap e1: no daemons active 2024-03-11T20:28:18.426 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:18 smithi083 ceph-mon[24410]: from='client.? 172.21.15.83:0/1614513426' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-11T20:28:18.771 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-11T20:28:18.772 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-11T20:28:18.773 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-11T20:28:19.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:19 smithi083 ceph-mon[24410]: from='client.? 172.21.15.83:0/3456670545' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-11T20:28:19.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:19 smithi083 ceph-mon[24410]: from='client.? 172.21.15.83:0/3456670545' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-11T20:28:19.359 INFO:teuthology.orchestra.run.smithi083.stderr:Generating new minimal ceph.conf... 2024-03-11T20:28:20.321 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 ceph-mon[24410]: from='client.? 172.21.15.83:0/4008626985' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:28:20.323 INFO:teuthology.orchestra.run.smithi083.stderr:Restarting the monitor... 2024-03-11T20:28:20.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:28:20.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 bash[24629]: Error: no container with name or ID "ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon.smithi083" found: no such container 2024-03-11T20:28:20.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[24406]: 2024-03-11T20:28:20.525+0000 7f24857e7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:28:20.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[24406]: 2024-03-11T20:28:20.525+0000 7f24857e7700 -1 mon.smithi083@0(leader) e1 *** Got Signal Terminated *** 2024-03-11T20:28:20.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:20 smithi083 podman[24642]: 2024-03-11 20:28:20.709297309 +0000 UTC m=+0.320226842 container died 213fa75f7953d262d76850a91b5684173803f9a347e3c6f3480284058146d2c4 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container) 2024-03-11T20:28:21.299 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24642]: 2024-03-11 20:28:21.07509585 +0000 UTC m=+0.686025444 container cleanup 213fa75f7953d262d76850a91b5684173803f9a347e3c6f3480284058146d2c4 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, vendor=Red Hat, Inc., architecture=x86_64, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, ceph=True, GIT_CLEAN=True, version=8, distribution-scope=public) 2024-03-11T20:28:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 bash[24642]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083 2024-03-11T20:28:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 bash[24668]: Error: no container with name or ID "ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon.smithi083" found: no such container 2024-03-11T20:28:21.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24655]: 2024-03-11 20:28:21.317478272 +0000 UTC m=+0.610272860 container remove 213fa75f7953d262d76850a91b5684173803f9a347e3c6f3480284058146d2c4 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, architecture=x86_64, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8) 2024-03-11T20:28:21.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service: Deactivated successfully. 2024-03-11T20:28:21.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 systemd[1]: Stopped Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:28:21.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 systemd[1]: Starting Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:28:21.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24736]: 2024-03-11 20:28:21.534972622 +0000 UTC m=+0.109652968 container create ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, com.redhat.component=centos-stream-container, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream) 2024-03-11T20:28:21.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24736]: 2024-03-11 20:28:21.443565319 +0000 UTC m=+0.018245674 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-03-11T20:28:22.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24736]: 2024-03-11 20:28:21.951062656 +0000 UTC m=+0.525743011 container init ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, GIT_BRANCH=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 podman[24736]: 2024-03-11 20:28:21.954315189 +0000 UTC m=+0.528995536 container start ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vcs-type=git, distribution-scope=public, io.buildah.version=1.19.8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.expose-services=, version=8) 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: set uid:gid to 167:167 (ceph:ceph) 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: pidfile_write: ignore empty --pid-file 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: load: jerasure load: lrc 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: RocksDB version: 6.15.5 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Compile date Apr 18 2022 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: DB SUMMARY 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: DB Session ID: F5S36WGGJ1NGMH9KB39I 2024-03-11T20:28:22.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: CURRENT file: CURRENT 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: IDENTITY file: IDENTITY 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi083/store.db dir, Total Num: 1, files: 000008.sst 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi083/store.db: 000010.log size: 82715 ; 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.error_if_exists: 0 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.create_if_missing: 0 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.paranoid_checks: 1 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.env: 0x5638da6c6860 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.fs: Posix File System 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.info_log: 0x5638dcb43fa0 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_file_opening_threads: 16 2024-03-11T20:28:22.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.statistics: (nil) 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.use_fsync: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_log_file_size: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.keep_log_file_num: 1000 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.recycle_log_file_num: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_fallocate: 1 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_mmap_reads: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_mmap_writes: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.use_direct_reads: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.create_missing_column_families: 0 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.db_log_dir: 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi083/store.db 2024-03-11T20:28:22.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-11T20:28:22.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.advise_random_on_open: 1 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.db_write_buffer_size: 0 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.write_buffer_manager: 0x5638dcc34330 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-11T20:28:22.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.rate_limiter: (nil) 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.wal_recovery_mode: 2 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.enable_thread_tracking: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.enable_pipelined_write: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.unordered_write: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.row_cache: None 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.wal_filter: None 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-11T20:28:22.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_ingest_behind: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.preserve_deletes: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.two_write_queues: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.manual_wal_flush: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.atomic_flush: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.log_readahead_size: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.best_efforts_recovery: 0 2024-03-11T20:28:22.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.allow_data_in_errors: 0 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.db_host_id: __hostname__ 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_background_jobs: 2 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_background_compactions: -1 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_subcompactions: 1 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_total_wal_size: 0 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-11T20:28:22.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_open_files: -1 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bytes_per_sync: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_readahead_size: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_background_flushes: -1 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Compression algorithms supported: 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kZSTD supported: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kXpressCompression supported: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kLZ4HCCompression supported: 1 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kLZ4Compression supported: 1 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kBZip2Compression supported: 0 2024-03-11T20:28:22.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kZlibCompression supported: 1 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: kSnappyCompression supported: 1 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.merge_operator: 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_filter: None 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_filter_factory: None 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.sst_partitioner_factory: None 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5638dcb0fd00) 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-03-11T20:28:22.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_type: 0 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_index_type: 0 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_shortening: 1 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: hash_index_allow_collision: 1 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: checksum: 1 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: no_block_cache: 0 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache: 0x5638dcb7a170 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_options: 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: capacity : 536870912 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_shard_bits : 4 2024-03-11T20:28:22.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: strict_capacity_limit : 0 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_compressed: (nil) 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: persistent_cache: (nil) 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size: 4096 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size_deviation: 10 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_restart_interval: 16 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_block_restart_interval: 1 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_block_size: 4096 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: partition_filters: 0 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: use_delta_encoding: 1 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: whole_key_filtering: 1 2024-03-11T20:28:22.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: verify_compression: 0 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: format_version: 4 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_index_compression: 1 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_align: 0 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.write_buffer_size: 33554432 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_write_buffer_number: 2 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression: NoCompression 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression: Disabled 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.prefix_extractor: nullptr 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.num_levels: 7 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-11T20:28:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.level: 32767 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.strategy: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compression_opts.enabled: false 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-11T20:28:22.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.target_file_size_base: 67108864 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-11T20:28:22.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.arena_block_size: 4194304 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.disable_auto_compactions: 0 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-11T20:28:22.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.table_properties_collectors: 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.inplace_update_support: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.bloom_locality: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.max_successive_merges: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.paranoid_file_checks: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.force_consistency_checks: 1 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.report_bg_io_stats: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.ttl: 2592000 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-11T20:28:22.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.enable_blob_files: false 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.min_blob_size: 0 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.blob_file_size: 268435456 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710188901980410, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710188901981099, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1710188901, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "208dcb78-0289-4857-9f4e-7d78f2636466", "db_session_id": "F5S36WGGJ1NGMH9KB39I"}} 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-03-11T20:28:22.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710188901981453, "job": 1, "event": "recovery_finished"} 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi083/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5638dcb60a80 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: DB pointer 0x5638dcb70000 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** DB Stats ** 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:28:22.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-11T20:28:22.204 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 26.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 26.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 163.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 25.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:28:22.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: starting mon.smithi083 rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi083 fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???) e1 preinit fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).mds e1 new map 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).mds e1 print_map 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: e1 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: legacy client fscid: -1 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: No filesystems configured 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:28:22.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: mon.smithi083@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expand map: {default=false} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta from 'false' to 'false' 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expanded map: {default=false} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expand map: {default=info} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta from 'info' to 'info' 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expanded map: {default=info} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expand map: {default=daemon} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta from 'daemon' to 'daemon' 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expanded map: {default=daemon} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expand map: {default=debug} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta from 'debug' to 'debug' 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:21 smithi083 ceph-mon[24755]: expand_channel_meta expanded map: {default=debug} 2024-03-11T20:28:22.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 ceph-mon[24755]: mon.smithi083 is new leader, mons smithi083 in quorum (ranks 0) 2024-03-11T20:28:22.209 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 ceph-mon[24755]: monmap e1: 1 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-03-11T20:28:22.209 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 ceph-mon[24755]: fsmap 2024-03-11T20:28:22.209 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 ceph-mon[24755]: osdmap e1: 0 total, 0 up, 0 in 2024-03-11T20:28:22.209 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 ceph-mon[24755]: mgrmap e1: no daemons active 2024-03-11T20:28:22.746 INFO:teuthology.orchestra.run.smithi083.stderr:Setting mon public_network to 172.21.0.0/20 2024-03-11T20:28:23.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 bash[24736]: ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 2024-03-11T20:28:23.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:22 smithi083 systemd[1]: Started Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:28:25.093 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote config to /etc/ceph/ceph.conf 2024-03-11T20:28:25.094 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:28:25.094 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mgr... 2024-03-11T20:28:25.094 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying port 9283 ... 2024-03-11T20:28:25.269 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mgr.smithi083.jyxogi.service: Unit ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mgr.smithi083.jyxogi.service not loaded. 2024-03-11T20:28:25.276 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715.target.wants/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mgr.smithi083.jyxogi.service → /etc/systemd/system/ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@.service. 2024-03-11T20:28:25.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:25 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4109250166' entity='client.admin' 2024-03-11T20:28:25.965 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-03-11T20:28:25.972 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:28:25.972 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-03-11T20:28:25.972 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-03-11T20:28:25.980 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-03-11T20:28:25.987 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:28:25.988 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-03-11T20:28:25.988 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-11T20:28:25.988 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr to start... 2024-03-11T20:28:25.988 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr... 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "c3f853a8-dfe5-11ee-95c8-87774f69a715", 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.744 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 4, 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-11T20:28:26.745 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-11T20:28:26.746 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-11T20:28:26.747 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-03-11T20:28:17.231307+0000", 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-11T20:28:26.748 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:28:27.037 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:26 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1173378625' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-11T20:28:27.344 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (1/15)... 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "c3f853a8-dfe5-11ee-95c8-87774f69a715", 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.885 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:29.886 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-11T20:28:29.887 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-11T20:28:29.887 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-11T20:28:29.887 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-11T20:28:29.890 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-11T20:28:29.890 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-11T20:28:29.891 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.892 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-03-11T20:28:17.231307+0000", 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:29.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:29.894 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-11T20:28:29.894 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:28:30.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:29 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/451285252' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-11T20:28:30.443 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (2/15)... 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: mgrmap e2: smithi083.jyxogi(active, starting, since 0.00262s) 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:28:32.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:32.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:31 smithi083 ceph-mon[24755]: from='mgr.14100 172.21.15.83:0/4055574280' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:33.111 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:33.111 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:28:33.111 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "c3f853a8-dfe5-11ee-95c8-87774f69a715", 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-03-11T20:28:33.112 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-03-11T20:28:33.113 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.114 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-03-11T20:28:17.231307+0000", 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-03-11T20:28:33.115 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-11T20:28:33.116 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:28:33.699 INFO:teuthology.orchestra.run.smithi083.stderr:mgr is available 2024-03-11T20:28:33.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:33 smithi083 ceph-mon[24755]: mgrmap e3: smithi083.jyxogi(active, since 1.00595s) 2024-03-11T20:28:33.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/217375761' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-11T20:28:34.272 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:34.272 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-03-11T20:28:34.272 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:34.272 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-11T20:28:34.272 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-11T20:28:34.273 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-11T20:28:34.853 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:34 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2075385201' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-11T20:28:34.856 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling cephadm module... 2024-03-11T20:28:35.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:35 smithi083 ceph-mon[24755]: mgrmap e4: smithi083.jyxogi(active, since 3s) 2024-03-11T20:28:35.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:35 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2650695347' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-11T20:28:36.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:36 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2650695347' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-11T20:28:36.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:36 smithi083 ceph-mon[24755]: mgrmap e5: smithi083.jyxogi(active, since 4s) 2024-03-11T20:28:36.937 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:28:36.938 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 5, 2024-03-11T20:28:36.938 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-03-11T20:28:36.938 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.jyxogi", 2024-03-11T20:28:36.938 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-11T20:28:36.938 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:28:37.536 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-03-11T20:28:37.536 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 5... 2024-03-11T20:28:37.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:37 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3874935963' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-11T20:28:42.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:28:42.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-03-11T20:28:42.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-03-11T20:28:42.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:28:42.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: osdmap e2: 0 total, 0 up, 0 in 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: mgrmap e6: smithi083.jyxogi(active, starting, since 0.0524357s) 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:42.823 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:42.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:42.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:28:42.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:28:42.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:42.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:42 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:43.085 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 5 is available 2024-03-11T20:28:43.086 INFO:teuthology.orchestra.run.smithi083.stderr:Setting orchestrator backend to cephadm... 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: [11/Mar/2024:20:28:42] ENGINE Bus STARTING 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: [11/Mar/2024:20:28:42] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: [11/Mar/2024:20:28:42] ENGINE Bus STARTED 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: mgrmap e7: smithi083.jyxogi(active, since 1.05499s) 2024-03-11T20:28:43.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:43 smithi083 ceph-mon[24755]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-11T20:28:44.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:44 smithi083 ceph-mon[24755]: mgrmap e8: smithi083.jyxogi(active, since 2s) 2024-03-11T20:28:44.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:44 smithi083 ceph-mon[24755]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:44.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:44 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:44.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:44 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:44.911 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: value unchanged 2024-03-11T20:28:45.505 INFO:teuthology.orchestra.run.smithi083.stderr:Generating ssh key... 2024-03-11T20:28:46.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:45 smithi083 ceph-mon[24755]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:47.575 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:47 smithi083 ceph-mon[24755]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:47.575 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:47 smithi083 ceph-mon[24755]: Generating ssh key... 2024-03-11T20:28:47.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:47 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:47.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:47 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:47.576 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnj1YKmmMdwNV/AaoOnZvLL69hLi3+XZ52WDdpzk62J/wcK8MEG7IJaAK5nKL8WyA+aP7NOs76HLRcs4UbMxJQ7O5ZljpLreDp+/2MKb+eGD+g0QD2rUeMVrdmKPNLV3nM7915/GRXlsxyz11I82+zFNdQh6HQ0CyRlj1SJG4/GZ2Oz1TIQMfOg9nK9kNrTQLtn/jG0iF91c8RN2+iqCJbJQGJguBcu6Moz9uRXBcaChegye1YN144oK3V8TJRcO1KKUHTnWdXPunDQQC/payP7yxcV1u0EL1gyhpwwrpkCAT1rKElPZR+OIbAqc/cUDO5DpVka2auAq3QLG8nLoFBseel+1v38VPRX6p8EkeKT7RCjtxf/iLVU7S3KFPsHXDGiF6V+7MpVuxn2ZIH66evzaBVN4f0k5gBzTyMSz2g04W+EY8tsX9OKlkdW0Zilu+PKc/6nmCGu5F2/QY705UCyqpnmTrplvyu1d7BXPtn5UueBXTXUaA7LRBiKLMEB+U= ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:28:48.118 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-11T20:28:48.119 INFO:teuthology.orchestra.run.smithi083.stderr:Adding key to root@localhost authorized_keys... 2024-03-11T20:28:48.119 INFO:teuthology.orchestra.run.smithi083.stderr:Adding host smithi083... 2024-03-11T20:28:48.361 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:48 smithi083 ceph-mon[24755]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:49.788 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Added host 'smithi083' with addr '172.21.15.83' 2024-03-11T20:28:49.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:49 smithi083 ceph-mon[24755]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi083", "addr": "172.21.15.83", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:49.799 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:49 smithi083 ceph-mon[24755]: Deploying cephadm binary to smithi083 2024-03-11T20:28:50.318 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mon service with default placement... 2024-03-11T20:28:50.982 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mon update... 2024-03-11T20:28:51.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:50 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:51.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:50 smithi083 ceph-mon[24755]: Added host smithi083 2024-03-11T20:28:51.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:50 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:28:51.540 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mgr service with default placement... 2024-03-11T20:28:52.128 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:51 smithi083 ceph-mon[24755]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:52.128 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:51 smithi083 ceph-mon[24755]: Saving service mon spec with placement count:5 2024-03-11T20:28:52.128 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:51 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:52.128 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-03-11T20:28:53.418 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:53 smithi083 ceph-mon[24755]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:53.418 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:53 smithi083 ceph-mon[24755]: Saving service mgr spec with placement count:2 2024-03-11T20:28:53.418 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:53 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:53.898 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying crash service with default placement... 2024-03-11T20:28:54.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:54 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:54.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:54 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:54.548 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled crash update... 2024-03-11T20:28:55.298 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying prometheus service with default placement... 2024-03-11T20:28:55.838 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:55 smithi083 ceph-mon[24755]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:55.838 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:55 smithi083 ceph-mon[24755]: Saving service crash spec with placement * 2024-03-11T20:28:55.838 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:55 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:56.243 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-03-11T20:28:56.840 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying grafana service with default placement... 2024-03-11T20:28:57.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:56 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:57.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:56 smithi083 ceph-mon[24755]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:57.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:56 smithi083 ceph-mon[24755]: Saving service prometheus spec with placement count:1 2024-03-11T20:28:57.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:56 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:57.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:56 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:57.611 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-03-11T20:28:58.198 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying node-exporter service with default placement... 2024-03-11T20:28:58.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:58 smithi083 ceph-mon[24755]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:28:58.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:58 smithi083 ceph-mon[24755]: Saving service grafana spec with placement count:1 2024-03-11T20:28:58.918 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:28:58 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:28:59.667 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-03-11T20:29:00.293 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying alertmanager service with default placement... 2024-03-11T20:29:00.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:00 smithi083 ceph-mon[24755]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:00.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:00 smithi083 ceph-mon[24755]: Saving service node-exporter spec with placement * 2024-03-11T20:29:00.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:00 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:00.924 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-03-11T20:29:02.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:01 smithi083 ceph-mon[24755]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:02.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:01 smithi083 ceph-mon[24755]: Saving service alertmanager spec with placement count:1 2024-03-11T20:29:02.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:01 smithi083 ceph-mon[24755]: from='mgr.14118 172.21.15.83:0/1818512814' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:02.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:01 smithi083 ceph-mon[24755]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:03.435 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:03 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1934010054' entity='client.admin' 2024-03-11T20:29:04.098 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling the dashboard module... 2024-03-11T20:29:04.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:04 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2183861567' entity='client.admin' 2024-03-11T20:29:04.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:04 smithi083 ceph-mon[24755]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:05.810 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:05 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/126294020' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-11T20:29:06.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:06 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/126294020' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-11T20:29:06.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:06 smithi083 ceph-mon[24755]: mgrmap e9: smithi083.jyxogi(active, since 24s) 2024-03-11T20:29:06.813 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:29:06.813 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 9, 2024-03-11T20:29:06.813 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-03-11T20:29:06.813 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.jyxogi", 2024-03-11T20:29:06.813 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-11T20:29:06.814 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:29:07.454 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-03-11T20:29:07.455 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 9... 2024-03-11T20:29:07.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:07 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2262678526' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-11T20:29:11.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:11 smithi083 ceph-mon[24755]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:29:11.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:11 smithi083 ceph-mon[24755]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:29:11.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:11 smithi083 ceph-mon[24755]: osdmap e3: 0 total, 0 up, 0 in 2024-03-11T20:29:12.337 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-03-11T20:29:12.337 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-03-11T20:29:12.338 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-03-11T20:29:12.338 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: mgrmap e10: smithi083.jyxogi(active, starting, since 0.055313s) 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:29:12.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:29:12.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:29:12.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:29:12.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:29:12.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:12 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:13.223 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 9 is available 2024-03-11T20:29:13.223 INFO:teuthology.orchestra.run.smithi083.stderr:Generating a dashboard self-signed certificate... 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: [11/Mar/2024:20:29:11] ENGINE Bus STARTING 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: [11/Mar/2024:20:29:11] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: [11/Mar/2024:20:29:11] ENGINE Bus STARTED 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: mgrmap e11: smithi083.jyxogi(active, since 1.06265s) 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-11T20:29:13.474 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:13 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:13.921 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Self-signed certificate created 2024-03-11T20:29:14.564 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial admin user... 2024-03-11T20:29:14.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:14 smithi083 ceph-mon[24755]: mgrmap e12: smithi083.jyxogi(active, since 2s) 2024-03-11T20:29:14.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:14 smithi083 ceph-mon[24755]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:14.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:14 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:14.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:14 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:14.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:14 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:15.508 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$/tq9I6OU4ovVIzWwML2vCuWxhd3tYmzzmUKI8wOBInjZV1MdqsD.a", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1710188955, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-11T20:29:16.333 INFO:teuthology.orchestra.run.smithi083.stderr:Fetching dashboard port number... 2024-03-11T20:29:16.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:16 smithi083 ceph-mon[24755]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:16.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:16 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:17.035 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 8443 2024-03-11T20:29:18.041 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-03-11T20:29:18.048 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-03-11T20:29:18.049 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-03-11T20:29:18.049 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph Dashboard is now available at: 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr: URL: https://smithi083.front.sepia.ceph.com:8443/ 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr: User: admin 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr: Password: xi4dc31geq 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:18.052 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling autotune for osd_memory_target 2024-03-11T20:29:18.319 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:17 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/429981651' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-11T20:29:19.375 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:18 smithi083 ceph-mon[24755]: mgrmap e13: smithi083.jyxogi(active, since 6s) 2024-03-11T20:29:20.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-03-11T20:29:20.734 INFO:teuthology.orchestra.run.smithi083.stderr:You can access the Ceph CLI with: 2024-03-11T20:29:20.734 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.734 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:29:20.734 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr:Please consider enabling telemetry to help improve Ceph: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: ceph telemetry on 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr:For more information see: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:29:20.735 INFO:teuthology.orchestra.run.smithi083.stderr:Bootstrap complete. 2024-03-11T20:29:20.775 INFO:tasks.cephadm:Fetching config... 2024-03-11T20:29:20.776 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:29:20.776 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-11T20:29:20.792 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-11T20:29:20.793 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:29:20.793 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-11T20:29:20.849 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-11T20:29:20.849 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:29:20.849 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/keyring of=/dev/stdout 2024-03-11T20:29:20.916 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-11T20:29:20.916 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:29:20.916 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-11T20:29:20.971 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-11T20:29:20.971 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnj1YKmmMdwNV/AaoOnZvLL69hLi3+XZ52WDdpzk62J/wcK8MEG7IJaAK5nKL8WyA+aP7NOs76HLRcs4UbMxJQ7O5ZljpLreDp+/2MKb+eGD+g0QD2rUeMVrdmKPNLV3nM7915/GRXlsxyz11I82+zFNdQh6HQ0CyRlj1SJG4/GZ2Oz1TIQMfOg9nK9kNrTQLtn/jG0iF91c8RN2+iqCJbJQGJguBcu6Moz9uRXBcaChegye1YN144oK3V8TJRcO1KKUHTnWdXPunDQQC/payP7yxcV1u0EL1gyhpwwrpkCAT1rKElPZR+OIbAqc/cUDO5DpVka2auAq3QLG8nLoFBseel+1v38VPRX6p8EkeKT7RCjtxf/iLVU7S3KFPsHXDGiF6V+7MpVuxn2ZIH66evzaBVN4f0k5gBzTyMSz2g04W+EY8tsX9OKlkdW0Zilu+PKc/6nmCGu5F2/QY705UCyqpnmTrplvyu1d7BXPtn5UueBXTXUaA7LRBiKLMEB+U= ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-11T20:29:21.049 INFO:teuthology.orchestra.run.smithi083.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnj1YKmmMdwNV/AaoOnZvLL69hLi3+XZ52WDdpzk62J/wcK8MEG7IJaAK5nKL8WyA+aP7NOs76HLRcs4UbMxJQ7O5ZljpLreDp+/2MKb+eGD+g0QD2rUeMVrdmKPNLV3nM7915/GRXlsxyz11I82+zFNdQh6HQ0CyRlj1SJG4/GZ2Oz1TIQMfOg9nK9kNrTQLtn/jG0iF91c8RN2+iqCJbJQGJguBcu6Moz9uRXBcaChegye1YN144oK3V8TJRcO1KKUHTnWdXPunDQQC/payP7yxcV1u0EL1gyhpwwrpkCAT1rKElPZR+OIbAqc/cUDO5DpVka2auAq3QLG8nLoFBseel+1v38VPRX6p8EkeKT7RCjtxf/iLVU7S3KFPsHXDGiF6V+7MpVuxn2ZIH66evzaBVN4f0k5gBzTyMSz2g04W+EY8tsX9OKlkdW0Zilu+PKc/6nmCGu5F2/QY705UCyqpnmTrplvyu1d7BXPtn5UueBXTXUaA7LRBiKLMEB+U= ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:29:21.071 DEBUG:teuthology.orchestra.run.smithi204:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnj1YKmmMdwNV/AaoOnZvLL69hLi3+XZ52WDdpzk62J/wcK8MEG7IJaAK5nKL8WyA+aP7NOs76HLRcs4UbMxJQ7O5ZljpLreDp+/2MKb+eGD+g0QD2rUeMVrdmKPNLV3nM7915/GRXlsxyz11I82+zFNdQh6HQ0CyRlj1SJG4/GZ2Oz1TIQMfOg9nK9kNrTQLtn/jG0iF91c8RN2+iqCJbJQGJguBcu6Moz9uRXBcaChegye1YN144oK3V8TJRcO1KKUHTnWdXPunDQQC/payP7yxcV1u0EL1gyhpwwrpkCAT1rKElPZR+OIbAqc/cUDO5DpVka2auAq3QLG8nLoFBseel+1v38VPRX6p8EkeKT7RCjtxf/iLVU7S3KFPsHXDGiF6V+7MpVuxn2ZIH66evzaBVN4f0k5gBzTyMSz2g04W+EY8tsX9OKlkdW0Zilu+PKc/6nmCGu5F2/QY705UCyqpnmTrplvyu1d7BXPtn5UueBXTXUaA7LRBiKLMEB+U= ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-11T20:29:21.118 INFO:teuthology.orchestra.run.smithi204.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnj1YKmmMdwNV/AaoOnZvLL69hLi3+XZ52WDdpzk62J/wcK8MEG7IJaAK5nKL8WyA+aP7NOs76HLRcs4UbMxJQ7O5ZljpLreDp+/2MKb+eGD+g0QD2rUeMVrdmKPNLV3nM7915/GRXlsxyz11I82+zFNdQh6HQ0CyRlj1SJG4/GZ2Oz1TIQMfOg9nK9kNrTQLtn/jG0iF91c8RN2+iqCJbJQGJguBcu6Moz9uRXBcaChegye1YN144oK3V8TJRcO1KKUHTnWdXPunDQQC/payP7yxcV1u0EL1gyhpwwrpkCAT1rKElPZR+OIbAqc/cUDO5DpVka2auAq3QLG8nLoFBseel+1v38VPRX6p8EkeKT7RCjtxf/iLVU7S3KFPsHXDGiF6V+7MpVuxn2ZIH66evzaBVN4f0k5gBzTyMSz2g04W+EY8tsX9OKlkdW0Zilu+PKc/6nmCGu5F2/QY705UCyqpnmTrplvyu1d7BXPtn5UueBXTXUaA7LRBiKLMEB+U= ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:29:21.133 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-11T20:29:21.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:21 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2913112370' entity='client.admin' 2024-03-11T20:29:24.380 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-11T20:29:24.380 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-11T20:29:24.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:24 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2489249639' entity='client.admin' 2024-03-11T20:29:27.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:26 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:27.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:26 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:29:27.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:26 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:27.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:26 smithi083 ceph-mon[24755]: Deploying daemon alertmanager.smithi083 on smithi083 2024-03-11T20:29:27.583 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi204 2024-03-11T20:29:27.584 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:29:27.584 DEBUG:teuthology.orchestra.run.smithi204:> dd of=/etc/ceph/ceph.conf 2024-03-11T20:29:27.602 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:29:27.602 DEBUG:teuthology.orchestra.run.smithi204:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:29:27.659 INFO:tasks.cephadm:Adding host smithi204 to orchestrator... 2024-03-11T20:29:27.659 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch host add smithi204 2024-03-11T20:29:28.338 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:28 smithi083 ceph-mon[24755]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:28.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:28 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:30.935 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:30 smithi083 ceph-mon[24755]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi204", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:30.935 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:30 smithi083 ceph-mon[24755]: Deploying cephadm binary to smithi204 2024-03-11T20:29:31.069 INFO:teuthology.orchestra.run.smithi083.stdout:Added host 'smithi204' with addr '172.21.15.204' 2024-03-11T20:29:32.227 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch host ls --format=json 2024-03-11T20:29:32.435 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:32 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:32.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:32 smithi083 ceph-mon[24755]: Added host smithi204 2024-03-11T20:29:32.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:32 smithi083 ceph-mon[24755]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:34.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:34 smithi083 ceph-mon[24755]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:36.365 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:29:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:[{"addr": "172.21.15.83", "hostname": "smithi083", "labels": [], "status": ""}, {"addr": "172.21.15.204", "hostname": "smithi204", "labels": [], "status": ""}] 2024-03-11T20:29:36.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:36 smithi083 ceph-mon[24755]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:37.383 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-11T20:29:37.383 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd crush tunables default 2024-03-11T20:29:37.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:37 smithi083 ceph-mon[24755]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:29:38.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:38 smithi083 ceph-mon[24755]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: Deploying daemon crash.smithi083 on smithi083 2024-03-11T20:29:40.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:39 smithi083 ceph-mon[24755]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:40.820 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2024-03-11T20:29:41.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:40 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2180112675' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-11T20:29:41.454 INFO:tasks.cephadm:Adding mon.smithi083 on smithi083 2024-03-11T20:29:41.454 INFO:tasks.cephadm:Adding mon.smithi204 on smithi204 2024-03-11T20:29:41.454 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch apply mon '2;smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204' 2024-03-11T20:29:41.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:41 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2180112675' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-11T20:29:41.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:41 smithi083 ceph-mon[24755]: osdmap e4: 0 total, 0 up, 0 in 2024-03-11T20:29:41.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:41 smithi083 ceph-mon[24755]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:41.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:41 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:43.296 INFO:teuthology.orchestra.run.smithi204.stdout:Scheduled mon update... 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:43.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:43 smithi083 ceph-mon[24755]: Deploying daemon grafana.smithi083 on smithi083 2024-03-11T20:29:43.912 DEBUG:teuthology.orchestra.run.smithi204:mon.smithi204> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi204.service 2024-03-11T20:29:43.914 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:29:43.915 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:29:44.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:44 smithi083 ceph-mon[24755]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:29:44.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:44 smithi083 ceph-mon[24755]: Saving service mon spec with placement smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204;count:2 2024-03-11T20:29:44.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:44 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:44.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:44 smithi083 ceph-mon[24755]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:45.772 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:29:45.772 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:29:45.774 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:29:46.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:46 smithi083 ceph-mon[24755]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:46.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:46 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1591007632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:29:46.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:46 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:47.346 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:29:47.346 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:29:48.435 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:48 smithi083 ceph-mon[24755]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:49.109 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:29:49.109 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:29:49.111 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:29:49.435 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:49 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2639388161' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:29:50.436 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:50 smithi083 ceph-mon[24755]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:50.655 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:29:50.655 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:29:52.561 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:29:52.562 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:29:52.564 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:29:52.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:52 smithi083 ceph-mon[24755]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:53.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:53 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/198177963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:29:54.119 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:29:54.119 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:29:54.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:54 smithi083 ceph-mon[24755]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:55.859 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:29:55.860 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:29:55.862 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:29:56.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:56 smithi083 ceph-mon[24755]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:56.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:56 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/654629574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:29:57.371 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:29:57.371 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:29:58.315 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:58 smithi083 ceph-mon[24755]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:29:59.323 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:29:59.323 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:29:59.324 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:29:59.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:59 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:29:59.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:59 smithi083 ceph-mon[24755]: Deploying daemon node-exporter.smithi083 on smithi083 2024-03-11T20:29:59.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:59 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/469383157' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:29:59.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:29:59 smithi083 ceph-mon[24755]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:00.930 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:00.930 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:02.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:02 smithi083 ceph-mon[24755]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:02.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:02 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:02.771 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:02.771 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:02.773 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:03.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:03 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:03.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:03 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-11T20:30:03.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:03 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1626966900' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:03.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:03 smithi083 ceph-mon[24755]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:04.373 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:04.374 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:04.877 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:04 smithi083 ceph-mon[24755]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-11T20:30:04.877 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:04 smithi083 ceph-mon[24755]: mgrmap e14: smithi083.jyxogi(active, since 52s) 2024-03-11T20:30:06.259 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:06.260 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:06.261 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:06.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:06 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1519938141' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:07.779 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:07.779 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:09.776 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:09.777 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:09.778 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:10.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:30:10.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: osdmap e5: 0 total, 0 up, 0 in 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: mgrmap e15: smithi083.jyxogi(active, starting, since 0.0533481s) 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:30:10.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:30:10.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1160011179' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:10.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:11.390 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:11.390 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:11.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: [11/Mar/2024:20:30:10] ENGINE Bus STARTING 2024-03-11T20:30:11.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: [11/Mar/2024:20:30:10] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:30:11.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: [11/Mar/2024:20:30:10] ENGINE Bus STARTED 2024-03-11T20:30:11.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:11.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: mgrmap e16: smithi083.jyxogi(active, since 1.0553s) 2024-03-11T20:30:11.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:11.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:13.244 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:13.245 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:13.247 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:13.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:13 smithi083 ceph-mon[24755]: mgrmap e17: smithi083.jyxogi(active, since 2s) 2024-03-11T20:30:13.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:13.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:13 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/3882963574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:14.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:14 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:14.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:14 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:30:14.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:14 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:30:14.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:14 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:15.216 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:15.216 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:15.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:15 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:30:17.618 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:17.619 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:17.620 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:17.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi204", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:17.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:17 smithi083 ceph-mon[24755]: Deploying daemon crash.smithi204 on smithi204 2024-03-11T20:30:18.887 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:18 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1709715033' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:19.237 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:19.238 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:21.627 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:21.627 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:21.628 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:30:22.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:22 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/3047530012' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:23.248 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:23.248 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:25.190 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:25.191 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:25.192 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:25.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:25 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:25.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:25 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2094899659' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:27.185 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:27.186 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:27.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:27.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:30:27.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:27.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:26 smithi083 ceph-mon[24755]: Deploying daemon mon.smithi204 on smithi204 2024-03-11T20:30:29.352 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:29 smithi204 systemd[1]: Starting Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:30:30.257 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:30.257 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":1,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:28:04.731740Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-11T20:30:30.258 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 1 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 podman[27222]: 2024-03-11 20:30:30.156391746 +0000 UTC m=+0.115381838 container create ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 podman[27222]: 2024-03-11 20:30:30.0675617 +0000 UTC m=+0.026551789 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 podman[27222]: 2024-03-11 20:30:30.367925058 +0000 UTC m=+0.326915150 container init ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., architecture=x86_64, version=8, RELEASE=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 podman[27222]: 2024-03-11 20:30:30.373363957 +0000 UTC m=+0.332354051 container start ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, name=centos-stream, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc.) 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: set uid:gid to 167:167 (ceph:ceph) 2024-03-11T20:30:30.403 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pidfile_write: ignore empty --pid-file 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: load: jerasure load: lrc 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: RocksDB version: 6.15.5 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Compile date Apr 18 2022 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: DB SUMMARY 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: DB Session ID: BLCSREV8DF6BS0ULG7NO 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: CURRENT file: CURRENT 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: IDENTITY file: IDENTITY 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi204/store.db dir, Total Num: 0, files: 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi204/store.db: 000004.log size: 511 ; 2024-03-11T20:30:30.404 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.error_if_exists: 0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.create_if_missing: 0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.paranoid_checks: 1 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.env: 0x564c398c6860 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.fs: Posix File System 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.info_log: 0x564c3ab53fa0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_file_opening_threads: 16 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.statistics: (nil) 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.use_fsync: 0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_log_file_size: 0 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-11T20:30:30.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.keep_log_file_num: 1000 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.recycle_log_file_num: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_fallocate: 1 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_mmap_reads: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_mmap_writes: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.use_direct_reads: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.create_missing_column_families: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.db_log_dir: 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi204/store.db 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-11T20:30:30.406 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.advise_random_on_open: 1 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.db_write_buffer_size: 0 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.write_buffer_manager: 0x564c3ac44330 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.rate_limiter: (nil) 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-11T20:30:30.407 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.wal_recovery_mode: 2 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.enable_thread_tracking: 0 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.enable_pipelined_write: 0 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.unordered_write: 0 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.row_cache: None 2024-03-11T20:30:30.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.wal_filter: None 2024-03-11T20:30:30.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-11T20:30:30.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_ingest_behind: 0 2024-03-11T20:30:30.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.preserve_deletes: 0 2024-03-11T20:30:30.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.two_write_queues: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.manual_wal_flush: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.atomic_flush: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.log_readahead_size: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.best_efforts_recovery: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.allow_data_in_errors: 0 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.db_host_id: __hostname__ 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_background_jobs: 2 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_background_compactions: -1 2024-03-11T20:30:30.411 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_subcompactions: 1 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_total_wal_size: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_open_files: -1 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bytes_per_sync: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_readahead_size: 0 2024-03-11T20:30:30.412 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_background_flushes: -1 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Compression algorithms supported: 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kZSTD supported: 0 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kXpressCompression supported: 0 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kLZ4HCCompression supported: 1 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kLZ4Compression supported: 1 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kBZip2Compression supported: 0 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kZlibCompression supported: 1 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: kSnappyCompression supported: 1 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi204/store.db/MANIFEST-000003 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-11T20:30:30.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.merge_operator: 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_filter: None 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_filter_factory: None 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.sst_partitioner_factory: None 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564c3ab1fd00) 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: cache_index_and_filter_blocks: 1 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: pin_top_level_index_and_filter: 1 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_type: 0 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: data_block_index_type: 0 2024-03-11T20:30:30.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_shortening: 1 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: hash_index_allow_collision: 1 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: checksum: 1 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: no_block_cache: 0 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache: 0x564c3ab8a170 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_name: BinnedLRUCache 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_options: 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: capacity : 536870912 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: num_shard_bits : 4 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: strict_capacity_limit : 0 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: high_pri_pool_ratio: 0.000 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_compressed: (nil) 2024-03-11T20:30:30.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: persistent_cache: (nil) 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_size: 4096 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_size_deviation: 10 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_restart_interval: 16 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_block_restart_interval: 1 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: metadata_block_size: 4096 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: partition_filters: 0 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: use_delta_encoding: 1 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: whole_key_filtering: 1 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: verify_compression: 0 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: read_amp_bytes_per_bit: 0 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: format_version: 4 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: enable_index_compression: 1 2024-03-11T20:30:30.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_align: 0 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.write_buffer_size: 33554432 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_write_buffer_number: 2 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression: NoCompression 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression: Disabled 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.prefix_extractor: nullptr 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.num_levels: 7 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-11T20:30:30.417 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.level: 32767 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.strategy: 0 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compression_opts.enabled: false 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-11T20:30:30.418 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.target_file_size_base: 67108864 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-11T20:30:30.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.arena_block_size: 4194304 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.disable_auto_compactions: 0 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-11T20:30:30.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.table_properties_collectors: 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.inplace_update_support: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.bloom_locality: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.max_successive_merges: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.paranoid_file_checks: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.force_consistency_checks: 1 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.report_bg_io_stats: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.ttl: 2592000 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-11T20:30:30.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.enable_blob_files: false 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.min_blob_size: 0 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.blob_file_size: 268435456 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi204/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189030398895, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-11T20:30:30.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189030399218, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1710189030, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bb54a397-7188-4558-bd37-3e87191547f2", "db_session_id": "BLCSREV8DF6BS0ULG7NO"}} 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189030399522, "job": 1, "event": "recovery_finished"} 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi204/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564c3ab70a80 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: DB pointer 0x564c3ab80000 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** DB Stats ** 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-11T20:30:30.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:30:30.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:30:30.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.9 0.00 0.00 1 0.000 0 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:30:30.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204 does not exist in monmap, will attempt to join an existing cluster 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: using public_addr v2:172.21.15.204:0/0 -> [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: starting mon.smithi204 rank -1 at public addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] at bind addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi204 fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(???) e0 preinit fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).mds e1 new map 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).mds e1 print_map 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: e1 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-11T20:30:30.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: legacy client fscid: -1 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: No filesystems configured 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-03-11T20:30:30.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon crash.smithi083 on smithi083 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2180112675' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-11T20:30:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2180112675' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: osdmap e4: 0 total, 0 up, 0 in 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon grafana.smithi083 on smithi083 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:30:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Saving service mon spec with placement smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204;count:2 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1591007632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2639388161' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/198177963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/654629574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon node-exporter.smithi083 on smithi083 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/469383157' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1626966900' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:30.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14160 172.21.15.83:0/503344030' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mgrmap e14: smithi083.jyxogi(active, since 52s) 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1519938141' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: osdmap e5: 0 total, 0 up, 0 in 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mgrmap e15: smithi083.jyxogi(active, starting, since 0.0533481s) 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:30:30.433 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1160011179' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: [11/Mar/2024:20:30:10] ENGINE Bus STARTING 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: [11/Mar/2024:20:30:10] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: [11/Mar/2024:20:30:10] ENGINE Bus STARTED 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mgrmap e16: smithi083.jyxogi(active, since 1.0553s) 2024-03-11T20:30:30.434 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mgrmap e17: smithi083.jyxogi(active, since 2s) 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/3882963574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi204", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.435 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon crash.smithi204 on smithi204 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1709715033' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/3047530012' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.436 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2094899659' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: Deploying daemon mon.smithi204 on smithi204 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-03-11T20:30:30.437 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:30 smithi204 ceph-mon[27258]: mon.smithi204@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-03-11T20:30:31.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:31 smithi204 bash[27222]: ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 2024-03-11T20:30:31.414 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:31 smithi204 systemd[1]: Started Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:30:32.004 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-11T20:30:32.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mon dump -f json 2024-03-11T20:30:35.766 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:30:35.766 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: mon.smithi083 calling monitor election 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: mon.smithi204 calling monitor election 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} 2024-03-11T20:30:35.767 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: fsmap 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: osdmap e5: 0 total, 0 up, 0 in 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: mgrmap e17: smithi083.jyxogi(active, since 25s) 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: overall HEALTH_OK 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:30:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: mon.smithi083 calling monitor election 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: mon.smithi204 calling monitor election 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: fsmap 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: osdmap e5: 0 total, 0 up, 0 in 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: mgrmap e17: smithi083.jyxogi(active, since 25s) 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: overall HEALTH_OK 2024-03-11T20:30:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:30:35.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1094404053' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:30:35.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:36.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:36 smithi204 ceph-mon[27258]: Deploying daemon node-exporter.smithi204 on smithi204 2024-03-11T20:30:36.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:36 smithi204 ceph-mon[27258]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:36.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:36 smithi204 ceph-mon[27258]: mgrmap e18: smithi083.jyxogi(active, since 26s), standbys: smithi204.qknpir 2024-03-11T20:30:36.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:30:36.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:36.926 INFO:teuthology.orchestra.run.smithi204.stdout: 2024-03-11T20:30:36.926 INFO:teuthology.orchestra.run.smithi204.stdout:{"epoch":2,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","modified":"2024-03-11T20:30:30.423451Z","created":"2024-03-11T20:28:04.731740Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi204","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:3300","nonce":0},{"type":"v1","addr":"172.21.15.204:6789","nonce":0}]},"addr":"172.21.15.204:6789/0","public_addr":"172.21.15.204:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-11T20:30:36.927 INFO:teuthology.orchestra.run.smithi204.stderr:dumped monmap epoch 2 2024-03-11T20:30:36.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:36 smithi083 ceph-mon[24755]: Deploying daemon node-exporter.smithi204 on smithi204 2024-03-11T20:30:36.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:36 smithi083 ceph-mon[24755]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:36.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:36 smithi083 ceph-mon[24755]: mgrmap e18: smithi083.jyxogi(active, since 26s), standbys: smithi204.qknpir 2024-03-11T20:30:36.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:30:36.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:30:37.747 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-11T20:30:37.747 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph config generate-minimal-conf 2024-03-11T20:30:37.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:37 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2531008476' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:37.919 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:37 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2531008476' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-11T20:30:38.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:38 smithi083 ceph-mon[24755]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:38.919 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:38 smithi204 ceph-mon[27258]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:39.711 INFO:teuthology.orchestra.run.smithi083.stdout:# minimal ceph.conf for c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:30:39.712 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2024-03-11T20:30:39.712 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:30:39.712 INFO:teuthology.orchestra.run.smithi083.stdout: mon_host = [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] 2024-03-11T20:30:40.321 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-11T20:30:40.321 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:30:40.321 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.conf 2024-03-11T20:30:40.391 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:30:40.391 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:30:40.458 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:30:40.458 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/ceph/ceph.conf 2024-03-11T20:30:40.492 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:30:40.492 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:30:40.563 INFO:tasks.cephadm:Deploying OSDs... 2024-03-11T20:30:40.563 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:30:40.563 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2024-03-11T20:30:40.578 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-11T20:30:40.578 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-03-11 20:29:59.274630895 +0000 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-03-11 20:26:58.981032206 +0000 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-03-11 20:26:58.981032206 +0000 2024-03-11T20:30:40.634 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-03-11 20:26:58.981032206 +0000 2024-03-11T20:30:40.635 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-11T20:30:40.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:40 smithi083 ceph-mon[24755]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:40.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:40.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:40 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/299531486' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:40.699 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-03-11T20:30:40.699 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-03-11T20:30:40.700 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000216145 s, 2.4 MB/s 2024-03-11T20:30:40.700 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-11T20:30:40.756 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-03-11 20:29:59.274630895 +0000 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-03-11 20:26:59.331023462 +0000 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-03-11 20:26:59.331023462 +0000 2024-03-11T20:30:40.812 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-03-11 20:26:59.331023462 +0000 2024-03-11T20:30:40.812 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-11T20:30:40.877 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-03-11T20:30:40.878 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-03-11T20:30:40.878 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000176044 s, 2.9 MB/s 2024-03-11T20:30:40.879 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-11T20:30:40.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:40 smithi204 ceph-mon[27258]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:40.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:40.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:40 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/299531486' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:40.935 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2024-03-11T20:30:40.990 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-03-11 20:29:59.274630895 +0000 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-03-11 20:26:59.687014567 +0000 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-03-11 20:26:59.687014567 +0000 2024-03-11T20:30:40.991 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-03-11 20:26:59.687014567 +0000 2024-03-11T20:30:40.991 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-11T20:30:41.056 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-03-11T20:30:41.056 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-03-11T20:30:41.056 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000174241 s, 2.9 MB/s 2024-03-11T20:30:41.057 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-11T20:30:41.113 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2024-03-11T20:30:41.166 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-03-11 20:29:59.275630870 +0000 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-03-11 20:27:00.035005872 +0000 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-03-11 20:27:00.035005872 +0000 2024-03-11T20:30:41.167 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-03-11 20:27:00.035005872 +0000 2024-03-11T20:30:41.167 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-11T20:30:41.232 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-03-11T20:30:41.232 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-03-11T20:30:41.232 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000188025 s, 2.7 MB/s 2024-03-11T20:30:41.233 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-11T20:30:41.289 DEBUG:teuthology.orchestra.run.smithi204:> set -ex 2024-03-11T20:30:41.289 DEBUG:teuthology.orchestra.run.smithi204:> dd if=/scratch_devs of=/dev/stdout 2024-03-11T20:30:41.305 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-11T20:30:41.305 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_1 2024-03-11T20:30:41.360 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-11T20:30:41.360 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:41.360 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-03-11T20:30:41.360 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:41.360 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:41.361 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-03-11 20:30:36.068734144 +0000 2024-03-11T20:30:41.361 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-03-11 20:26:59.648178947 +0000 2024-03-11T20:30:41.361 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-03-11 20:26:59.648178947 +0000 2024-03-11T20:30:41.361 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-03-11 20:26:59.648178947 +0000 2024-03-11T20:30:41.361 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-11T20:30:41.425 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-03-11T20:30:41.426 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-03-11T20:30:41.426 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000272633 s, 1.9 MB/s 2024-03-11T20:30:41.427 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-11T20:30:41.482 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_2 2024-03-11T20:30:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-11T20:30:41.537 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:41.537 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-03-11T20:30:41.537 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:41.538 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:41.538 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-03-11 20:30:36.069734129 +0000 2024-03-11T20:30:41.538 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-03-11 20:26:59.986173978 +0000 2024-03-11T20:30:41.538 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-03-11 20:26:59.986173978 +0000 2024-03-11T20:30:41.538 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-03-11 20:26:59.986173978 +0000 2024-03-11T20:30:41.538 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-11T20:30:41.603 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-03-11T20:30:41.604 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-03-11T20:30:41.604 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000215966 s, 2.4 MB/s 2024-03-11T20:30:41.605 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-11T20:30:41.661 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_3 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-03-11 20:30:36.069734129 +0000 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-03-11 20:27:00.344168716 +0000 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-03-11 20:27:00.344168716 +0000 2024-03-11T20:30:41.717 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-03-11 20:27:00.344168716 +0000 2024-03-11T20:30:41.717 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-11T20:30:41.783 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-03-11T20:30:41.783 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-03-11T20:30:41.783 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000177596 s, 2.9 MB/s 2024-03-11T20:30:41.784 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-11T20:30:41.840 DEBUG:teuthology.orchestra.run.smithi204:> stat /dev/vg_nvme/lv_4 2024-03-11T20:30:41.896 INFO:teuthology.orchestra.run.smithi204.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-11T20:30:41.896 INFO:teuthology.orchestra.run.smithi204.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-11T20:30:41.896 INFO:teuthology.orchestra.run.smithi204.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout:Context: system_u:object_r:device_t:s0 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout:Access: 2024-03-11 20:30:36.069734129 +0000 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout:Modify: 2024-03-11 20:27:00.682163747 +0000 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout:Change: 2024-03-11 20:27:00.682163747 +0000 2024-03-11T20:30:41.897 INFO:teuthology.orchestra.run.smithi204.stdout: Birth: 2024-03-11 20:27:00.682163747 +0000 2024-03-11T20:30:41.897 DEBUG:teuthology.orchestra.run.smithi204:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-11T20:30:41.962 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records in 2024-03-11T20:30:41.962 INFO:teuthology.orchestra.run.smithi204.stderr:1+0 records out 2024-03-11T20:30:41.962 INFO:teuthology.orchestra.run.smithi204.stderr:512 bytes copied, 0.000168376 s, 3.0 MB/s 2024-03-11T20:30:41.963 DEBUG:teuthology.orchestra.run.smithi204:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-11T20:30:42.019 INFO:tasks.cephadm:Deploying osd.0 on smithi083 with /dev/vg_nvme/lv_4... 2024-03-11T20:30:42.019 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-11T20:30:42.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:42 smithi083 ceph-mon[24755]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:42.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:42 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:42.919 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:42 smithi204 ceph-mon[27258]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:42.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:42 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:43.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:43 smithi083 ceph-mon[24755]: Deploying daemon prometheus.smithi083 on smithi083 2024-03-11T20:30:43.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:43 smithi204 ceph-mon[27258]: Deploying daemon prometheus.smithi083 on smithi083 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0142186 s, 737 MB/s 2024-03-11T20:30:44.157 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-03-11T20:30:44.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:44 smithi204 ceph-mon[27258]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:44.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:44.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:44 smithi083 ceph-mon[24755]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:44.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:45.231 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi083:vg_nvme/lv_4 2024-03-11T20:30:46.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:46 smithi204 ceph-mon[27258]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:46.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:46 smithi083 ceph-mon[24755]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:48.106 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:48 smithi083 ceph-mon[24755]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:48.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:48 smithi204 ceph-mon[27258]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:49.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:49 smithi083 ceph-mon[24755]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:30:49.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:30:49.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:30:49.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:49.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:49 smithi204 ceph-mon[27258]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:30:49.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:30:49.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:30:49.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:50.419 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:50 smithi204 ceph-mon[27258]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:50.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:50 smithi083 ceph-mon[24755]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:52.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:52 smithi204 ceph-mon[27258]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:52.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:52 smithi083 ceph-mon[24755]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:54.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:54 smithi204 ceph-mon[27258]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:54.937 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:54 smithi083 ceph-mon[24755]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:56.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:56 smithi204 ceph-mon[27258]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:56.929 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:56 smithi083 ceph-mon[24755]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:58.226 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:58 smithi083 ceph-mon[24755]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:58.377 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:58 smithi204 ceph-mon[27258]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:30:59.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:59.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:59.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:59.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:30:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:30:59.655 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:30:59.655 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:30:59.655 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:30:59.655 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:30:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2042039503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d63f4854-b6ea-43a1-8f55-a345f136033d"}]: dispatch 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2042039503' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d63f4854-b6ea-43a1-8f55-a345f136033d"}]': finished 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: osdmap e6: 1 total, 0 up, 1 in 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:00.574 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1588549624' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:31:00.575 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:00.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2042039503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d63f4854-b6ea-43a1-8f55-a345f136033d"}]: dispatch 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2042039503' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d63f4854-b6ea-43a1-8f55-a345f136033d"}]': finished 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: osdmap e6: 1 total, 0 up, 1 in 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1588549624' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:31:00.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:01.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:01 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:31:01.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:01 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:31:01.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:01.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:01 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:31:01.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:01 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:31:01.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:02.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:02 smithi083 ceph-mon[24755]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:02.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:02 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:02.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:02 smithi204 ceph-mon[27258]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:02.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:02 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:03.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:31:03.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:03.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:03.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:31:03.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:31:03.676 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:03 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:03.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:31:03.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:03.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:03.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:31:03.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:31:03.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:03 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:04.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:04 smithi083 ceph-mon[24755]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-03-11T20:31:04.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:04 smithi083 ceph-mon[24755]: Reconfiguring daemon mon.smithi083 on smithi083 2024-03-11T20:31:04.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:04 smithi083 ceph-mon[24755]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:04.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:04 smithi204 ceph-mon[27258]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-03-11T20:31:04.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:04 smithi204 ceph-mon[27258]: Reconfiguring daemon mon.smithi083 on smithi083 2024-03-11T20:31:04.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:04 smithi204 ceph-mon[27258]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:06.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:06.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: Reconfiguring mgr.smithi083.jyxogi (unknown last config time)... 2024-03-11T20:31:06.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:31:06.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:31:06.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:06.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: Reconfiguring daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:31:06.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:05 smithi204 ceph-mon[27258]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:06.437 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: Reconfiguring mgr.smithi083.jyxogi (unknown last config time)... 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: Reconfiguring daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:31:06.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:05 smithi083 ceph-mon[24755]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:07.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:07 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:31:07.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:07 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:07.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:07 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:31:07.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:07 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:08.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:08 smithi083 ceph-mon[24755]: Deploying daemon osd.0 on smithi083 2024-03-11T20:31:08.109 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:08 smithi083 ceph-mon[24755]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:08.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:08 smithi204 ceph-mon[27258]: Deploying daemon osd.0 on smithi083 2024-03-11T20:31:08.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:08 smithi204 ceph-mon[27258]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:09.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:09.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:09 smithi083 ceph-mon[24755]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-03-11T20:31:09.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:09 smithi083 ceph-mon[24755]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-03-11T20:31:09.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:09.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:09 smithi204 ceph-mon[27258]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-03-11T20:31:09.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:09 smithi204 ceph-mon[27258]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-03-11T20:31:10.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:10 smithi083 ceph-mon[24755]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:10.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:31:10.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:31:10.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:10 smithi204 ceph-mon[27258]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:10.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:31:10.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:31:12.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:12 smithi083 ceph-mon[24755]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:12.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:12 smithi204 ceph-mon[27258]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:14.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:14 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:14.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:14 smithi083 ceph-mon[24755]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:14.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:14 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:14.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:14 smithi204 ceph-mon[27258]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:16.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:16 smithi204 ceph-mon[27258]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:16.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:16 smithi083 ceph-mon[24755]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:17.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:17 smithi083 ceph-mon[24755]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-11T20:31:17.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:17 smithi204 ceph-mon[27258]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-11T20:31:18.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:18 smithi083 ceph-mon[24755]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:18.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:18 smithi083 ceph-mon[24755]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-11T20:31:18.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:18 smithi083 ceph-mon[24755]: osdmap e7: 1 total, 0 up, 1 in 2024-03-11T20:31:18.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:18.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:18 smithi083 ceph-mon[24755]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:31:18.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:18 smithi204 ceph-mon[27258]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:18.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:18 smithi204 ceph-mon[27258]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-11T20:31:18.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:18 smithi204 ceph-mon[27258]: osdmap e7: 1 total, 0 up, 1 in 2024-03-11T20:31:18.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:18.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:18 smithi204 ceph-mon[27258]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:31:19.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:31:19.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:31:19.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:31:19.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: osdmap e8: 1 total, 0 up, 1 in 2024-03-11T20:31:19.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:19.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:19.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:19.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:31:19.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: osdmap e8: 1 total, 0 up, 1 in 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:19.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:19.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:19.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:31:19.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:20.432 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 0 on host 'smithi083' 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: Reconfiguring crash.smithi083 (monmap changed)... 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: Reconfiguring daemon crash.smithi083 on smithi083 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932] boot 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: osdmap e9: 1 total, 1 up, 1 in 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:20.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:20 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:20.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: Reconfiguring crash.smithi083 (monmap changed)... 2024-03-11T20:31:20.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: Reconfiguring daemon crash.smithi083 on smithi083 2024-03-11T20:31:20.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-11T20:31:20.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: osd.0 [v2:172.21.15.83:6802/557961932,v1:172.21.15.83:6803/557961932] boot 2024-03-11T20:31:20.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: osdmap e9: 1 total, 1 up, 1 in 2024-03-11T20:31:20.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:31:20.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:20 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:21.007 DEBUG:teuthology.orchestra.run.smithi083:osd.0> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.0.service 2024-03-11T20:31:21.010 INFO:tasks.cephadm:Deploying osd.1 on smithi083 with /dev/vg_nvme/lv_3... 2024-03-11T20:31:21.010 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-11T20:31:21.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:21 smithi083 ceph-mon[24755]: osdmap e10: 1 total, 1 up, 1 in 2024-03-11T20:31:21.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:21 smithi204 ceph-mon[27258]: osdmap e10: 1 total, 1 up, 1 in 2024-03-11T20:31:22.903 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:22 smithi083 ceph-mon[24755]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:22.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:22 smithi204 ceph-mon[27258]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:24.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:24.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:24 smithi204 ceph-mon[27258]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-03-11T20:31:24.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:24 smithi204 ceph-mon[27258]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-03-11T20:31:24.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:24 smithi204 ceph-mon[27258]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:24.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:24.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:24 smithi083 ceph-mon[24755]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-03-11T20:31:24.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:24 smithi083 ceph-mon[24755]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-03-11T20:31:24.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:24 smithi083 ceph-mon[24755]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127621 s, 822 MB/s 2024-03-11T20:31:25.438 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-03-11T20:31:26.669 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:26 smithi083 ceph-mon[24755]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:26.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:26 smithi204 ceph-mon[27258]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:27.378 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi083:vg_nvme/lv_3 2024-03-11T20:31:28.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:28 smithi083 ceph-mon[24755]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:28.420 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:28 smithi204 ceph-mon[27258]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:30 smithi204 ceph-mon[27258]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:30 smithi204 ceph-mon[27258]: from='client.14258 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:31:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:31:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:31:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:30.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:30 smithi083 ceph-mon[24755]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:30.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:30 smithi083 ceph-mon[24755]: from='client.14258 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:31:30.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:31:30.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:31:30.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:32.920 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:32 smithi204 ceph-mon[27258]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:32.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:32 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:32.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:32 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:31:32.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:32 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:32.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:32 smithi083 ceph-mon[24755]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:32.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:32 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:32.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:32 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:31:32.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:32 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:33.816 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: Reconfiguring crash.smithi204 (monmap changed)... 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: Reconfiguring daemon crash.smithi204 on smithi204 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2781869144' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75c3c297-0eb7-4fd4-902a-3e69959edd14"}]: dispatch 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2781869144' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75c3c297-0eb7-4fd4-902a-3e69959edd14"}]': finished 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: osdmap e11: 2 total, 1 up, 2 in 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:33.817 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:33 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4134493042' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: Reconfiguring crash.smithi204 (monmap changed)... 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: Reconfiguring daemon crash.smithi204 on smithi204 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2781869144' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75c3c297-0eb7-4fd4-902a-3e69959edd14"}]: dispatch 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2781869144' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75c3c297-0eb7-4fd4-902a-3e69959edd14"}]': finished 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: osdmap e11: 2 total, 1 up, 2 in 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:31:33.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:31:33.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:33.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4134493042' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:31:34.615 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: Reconfiguring mgr.smithi204.qknpir (monmap changed)... 2024-03-11T20:31:34.615 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: Reconfiguring daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:31:34.616 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:34.616 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:34.616 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:31:34.616 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:31:34.616 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:34 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: Reconfiguring mgr.smithi204.qknpir (monmap changed)... 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: Reconfiguring daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:31:34.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:34 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:35.768 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:35 smithi204 ceph-mon[27258]: Reconfiguring mon.smithi204 (monmap changed)... 2024-03-11T20:31:35.769 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:35 smithi204 ceph-mon[27258]: Reconfiguring daemon mon.smithi204 on smithi204 2024-03-11T20:31:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:35 smithi083 ceph-mon[24755]: Reconfiguring mon.smithi204 (monmap changed)... 2024-03-11T20:31:35.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:35 smithi083 ceph-mon[24755]: Reconfiguring daemon mon.smithi204 on smithi204 2024-03-11T20:31:36.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:36 smithi083 ceph-mon[24755]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:36.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:36.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:31:36.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:36 smithi204 ceph-mon[27258]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:36.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:36.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:31:37.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:37 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:31:37.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:37 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:31:38.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:38 smithi204 ceph-mon[27258]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:38.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:38 smithi083 ceph-mon[24755]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:39.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:39 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:31:39.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:39 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:39.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:39 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:31:39.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:39 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:40.773 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:40 smithi083 ceph-mon[24755]: Deploying daemon osd.1 on smithi083 2024-03-11T20:31:40.773 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:40 smithi083 ceph-mon[24755]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:40.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:40 smithi204 ceph-mon[27258]: Deploying daemon osd.1 on smithi083 2024-03-11T20:31:40.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:40 smithi204 ceph-mon[27258]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:42 smithi083 ceph-mon[24755]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:42.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:42 smithi204 ceph-mon[27258]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:44.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:44 smithi204 ceph-mon[27258]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:44.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:44.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:44 smithi083 ceph-mon[24755]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:44.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:46.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:46 smithi204 ceph-mon[27258]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:46.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:46 smithi083 ceph-mon[24755]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:47.418 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 1 on host 'smithi083' 2024-03-11T20:31:48.073 DEBUG:teuthology.orchestra.run.smithi083:osd.1> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.1.service 2024-03-11T20:31:48.076 INFO:tasks.cephadm:Deploying osd.2 on smithi083 with /dev/vg_nvme/lv_2... 2024-03-11T20:31:48.076 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:31:48.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:48.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:48 smithi204 ceph-mon[27258]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-11T20:31:48.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-03-11T20:31:48.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:31:48.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:48.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:48 smithi083 ceph-mon[24755]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-11T20:31:49.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-mon[24755]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-11T20:31:49.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-mon[24755]: osdmap e12: 2 total, 1 up, 2 in 2024-03-11T20:31:49.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:49.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-mon[24755]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:31:49.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:49.689 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[38025]: 2024-03-11T20:31:49.478+0000 7fa9113e4700 -1 osd.1 0 waiting for initial osdmap 2024-03-11T20:31:49.689 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:31:49 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[38025]: 2024-03-11T20:31:49.481+0000 7fa90dd7f700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:31:49.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:49 smithi204 ceph-mon[27258]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-11T20:31:49.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:49 smithi204 ceph-mon[27258]: osdmap e12: 2 total, 1 up, 2 in 2024-03-11T20:31:49.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:49 smithi204 ceph-mon[27258]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:31:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:50.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:31:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: osdmap e13: 2 total, 1 up, 2 in 2024-03-11T20:31:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:50 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: from='osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: osdmap e13: 2 total, 1 up, 2 in 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-11T20:31:50.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:50 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:51.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:51 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:31:51.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:51 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:31:51.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:51 smithi083 ceph-mon[24755]: osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135] boot 2024-03-11T20:31:51.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:51 smithi083 ceph-mon[24755]: osdmap e14: 2 total, 2 up, 2 in 2024-03-11T20:31:51.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:51 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:51.653 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-11T20:31:51.654 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-11T20:31:51.654 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-03-11T20:31:51.654 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-03-11T20:31:51.654 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126039 s, 832 MB/s 2024-03-11T20:31:51.654 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-03-11T20:31:51.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:51 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:31:51.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:51 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:31:51.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:51 smithi204 ceph-mon[27258]: osd.1 [v2:172.21.15.83:6810/2904385135,v1:172.21.15.83:6811/2904385135] boot 2024-03-11T20:31:51.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:51 smithi204 ceph-mon[27258]: osdmap e14: 2 total, 2 up, 2 in 2024-03-11T20:31:51.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:51 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:31:52.425 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi083:vg_nvme/lv_2 2024-03-11T20:31:52.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:52 smithi083 ceph-mon[24755]: osdmap e15: 2 total, 2 up, 2 in 2024-03-11T20:31:52.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:52 smithi083 ceph-mon[24755]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:52.822 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:52 smithi204 ceph-mon[27258]: osdmap e15: 2 total, 2 up, 2 in 2024-03-11T20:31:52.822 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:52 smithi204 ceph-mon[27258]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:54.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:54 smithi083 ceph-mon[24755]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:54.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:54 smithi204 ceph-mon[27258]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:55.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:55 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:31:55.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:55 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:31:55.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:55 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:55.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:55 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:55.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:55 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:31:55.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:55 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:31:55.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:55 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:31:55.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:55 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:31:56.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:56 smithi083 ceph-mon[24755]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:31:56.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:56 smithi083 ceph-mon[24755]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:56.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:56 smithi204 ceph-mon[27258]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:31:56.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:56 smithi204 ceph-mon[27258]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:58.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:58 smithi083 ceph-mon[24755]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:58.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:58 smithi204 ceph-mon[27258]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:31:59.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:59 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2448489865' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b2be69-985a-4bc9-98de-17219ec4bb04"}]: dispatch 2024-03-11T20:31:59.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:59 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2448489865' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9b2be69-985a-4bc9-98de-17219ec4bb04"}]': finished 2024-03-11T20:31:59.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:59 smithi204 ceph-mon[27258]: osdmap e16: 3 total, 2 up, 3 in 2024-03-11T20:31:59.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:31:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:31:59.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:59 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2448489865' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9b2be69-985a-4bc9-98de-17219ec4bb04"}]: dispatch 2024-03-11T20:31:59.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:59 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2448489865' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9b2be69-985a-4bc9-98de-17219ec4bb04"}]': finished 2024-03-11T20:31:59.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:59 smithi083 ceph-mon[24755]: osdmap e16: 3 total, 2 up, 3 in 2024-03-11T20:31:59.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:31:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:00 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2311200516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:00 smithi083 ceph-mon[24755]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:00.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:00 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2311200516' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:00.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:00 smithi204 ceph-mon[27258]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:02.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:02 smithi083 ceph-mon[24755]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:02.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:02 smithi204 ceph-mon[27258]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:04.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:04 smithi083 ceph-mon[24755]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:04.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:04 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:04.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:04 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:04.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:04 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:04.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:04 smithi204 ceph-mon[27258]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:04.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:04 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:04.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:04 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:04.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:04 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi083 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi083 to 6432M 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:05.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:05.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi083 2024-03-11T20:32:05.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi083 to 6432M 2024-03-11T20:32:05.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:05.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:05.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:05.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:05.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:06 smithi083 ceph-mon[24755]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:32:06.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:06.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:06 smithi204 ceph-mon[27258]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:06.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:06.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:32:06.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:07.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:07 smithi204 ceph-mon[27258]: Deploying daemon osd.2 on smithi083 2024-03-11T20:32:07.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:07 smithi083 ceph-mon[24755]: Deploying daemon osd.2 on smithi083 2024-03-11T20:32:08.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:08 smithi204 ceph-mon[27258]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:08.938 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:08 smithi083 ceph-mon[24755]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:09.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:32:09.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:32:09.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:32:09.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:32:10.865 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:10 smithi083 ceph-mon[24755]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:10.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:10 smithi204 ceph-mon[27258]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:12.710 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:12 smithi083 ceph-mon[24755]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:12.710 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:12 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:12.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:12 smithi204 ceph-mon[27258]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:12.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:12 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:14.918 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:14 smithi083 ceph-mon[24755]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:14.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:14 smithi204 ceph-mon[27258]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:15.898 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:15.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:16.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:16.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:16.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:16.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:16.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:16.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:16.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:18.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:18 smithi204 ceph-mon[27258]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:18.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:18 smithi204 ceph-mon[27258]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-11T20:32:18.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:18 smithi083 ceph-mon[24755]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:18.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:18 smithi083 ceph-mon[24755]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-11T20:32:18.846 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 2 on host 'smithi083' 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: osdmap e17: 3 total, 2 up, 3 in 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:19.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:19.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-11T20:32:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: osdmap e17: 3 total, 2 up, 3 in 2024-03-11T20:32:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:32:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:19.565 DEBUG:teuthology.orchestra.run.smithi083:osd.2> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.2.service 2024-03-11T20:32:19.568 INFO:tasks.cephadm:Deploying osd.3 on smithi083 with /dev/vg_nvme/lv_1... 2024-03-11T20:32:19.568 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-11T20:32:20.064 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:20 smithi083 ceph-mon[24755]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:32:20.064 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:20 smithi083 ceph-mon[24755]: osdmap e18: 3 total, 2 up, 3 in 2024-03-11T20:32:20.064 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:20 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:20.064 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:20 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:20.064 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:20 smithi083 ceph-mon[24755]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:20.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:20 smithi204 ceph-mon[27258]: from='osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:32:20.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:20 smithi204 ceph-mon[27258]: osdmap e18: 3 total, 2 up, 3 in 2024-03-11T20:32:20.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:20 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:20.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:20 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:20.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:20 smithi204 ceph-mon[27258]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284] boot 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: osdmap e19: 3 total, 3 up, 3 in 2024-03-11T20:32:21.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:21 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:21.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:32:21.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:32:21.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:21.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: osd.2 [v2:172.21.15.83:6818/882489284,v1:172.21.15.83:6819/882489284] boot 2024-03-11T20:32:21.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: osdmap e19: 3 total, 3 up, 3 in 2024-03-11T20:32:21.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:21 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:32:22.323 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:22 smithi204 ceph-mon[27258]: osdmap e20: 3 total, 3 up, 3 in 2024-03-11T20:32:22.323 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:22 smithi204 ceph-mon[27258]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:22.323 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:22 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-11T20:32:22.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:22 smithi083 ceph-mon[24755]: osdmap e20: 3 total, 3 up, 3 in 2024-03-11T20:32:22.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:22 smithi083 ceph-mon[24755]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:22.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125088 s, 838 MB/s 2024-03-11T20:32:22.525 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-03-11T20:32:23.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:23 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-11T20:32:23.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:23 smithi204 ceph-mon[27258]: osdmap e21: 3 total, 3 up, 3 in 2024-03-11T20:32:23.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:23 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-11T20:32:23.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:23 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-11T20:32:23.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:23 smithi083 ceph-mon[24755]: osdmap e21: 3 total, 3 up, 3 in 2024-03-11T20:32:23.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:23 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-11T20:32:23.566 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi083:vg_nvme/lv_1 2024-03-11T20:32:23.733 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43168]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-11T20:32:23.733 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43168]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:23.733 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43168]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:24.016 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43191]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-11T20:32:24.016 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43191]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:24.016 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:32:23 smithi083 sudo[43191]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:24.268 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43252]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-11T20:32:24.269 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43252]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:24.269 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43252]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:24.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-11T20:32:24.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:24 smithi083 ceph-mon[24755]: osdmap e22: 3 total, 3 up, 3 in 2024-03-11T20:32:24.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:24 smithi083 ceph-mon[24755]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:24.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:24.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-11T20:32:24.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:24 smithi204 ceph-mon[27258]: osdmap e22: 3 total, 3 up, 3 in 2024-03-11T20:32:24.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:24 smithi204 ceph-mon[27258]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:24.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:24.526 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43276]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-11T20:32:24.526 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43276]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:24.527 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43276]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:24.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43290]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-11T20:32:24.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43290]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:24.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43290]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:25.038 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43304]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-11T20:32:25.038 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43295]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-11T20:32:25.039 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43295]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:25.039 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:32:24 smithi083 sudo[43295]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:25.296 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43304]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:25.296 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43304]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:25.557 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 ceph-mon[27258]: osdmap e23: 3 total, 3 up, 3 in 2024-03-11T20:32:25.558 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 ceph-mon[27258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-11T20:32:25.558 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29586]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-11T20:32:25.558 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29586]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:25.558 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29586]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:25.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43344]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-11T20:32:25.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43344]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:25.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 sudo[43344]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:25.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 ceph-mon[24755]: osdmap e23: 3 total, 3 up, 3 in 2024-03-11T20:32:25.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:25 smithi083 ceph-mon[24755]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-11T20:32:25.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29589]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-11T20:32:25.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29589]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-11T20:32:25.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:25 smithi204 sudo[29589]: pam_unix(sudo:session): session closed for user root 2024-03-11T20:32:26.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: mgrmap e19: smithi083.jyxogi(active, since 2m), standbys: smithi204.qknpir 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:26.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: mgrmap e19: smithi083.jyxogi(active, since 2m), standbys: smithi204.qknpir 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:26.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:27.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:27 smithi083 ceph-mon[24755]: from='client.14300 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:32:27.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:27 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:32:27.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:27 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:32:27.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:27 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:27.671 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:27 smithi204 ceph-mon[27258]: from='client.14300 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:32:27.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:27 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:32:27.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:27 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:32:27.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:27 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:28.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:28 smithi204 ceph-mon[27258]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:28.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:28 smithi083 ceph-mon[24755]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:30.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:30.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:30.921 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:30.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:31.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:31 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi083 to 4288M 2024-03-11T20:32:31.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:31 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1607340531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93517251-5f44-4d7b-8624-4b4c3f0a9f4b"}]: dispatch 2024-03-11T20:32:31.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:31 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1607340531' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93517251-5f44-4d7b-8624-4b4c3f0a9f4b"}]': finished 2024-03-11T20:32:31.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:31 smithi083 ceph-mon[24755]: osdmap e24: 4 total, 3 up, 4 in 2024-03-11T20:32:31.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:31 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:31.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:31 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi083 to 4288M 2024-03-11T20:32:31.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:31 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1607340531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "93517251-5f44-4d7b-8624-4b4c3f0a9f4b"}]: dispatch 2024-03-11T20:32:31.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:31 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1607340531' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "93517251-5f44-4d7b-8624-4b4c3f0a9f4b"}]': finished 2024-03-11T20:32:31.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:31 smithi204 ceph-mon[27258]: osdmap e24: 4 total, 3 up, 4 in 2024-03-11T20:32:31.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:31 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:32.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:32 smithi204 ceph-mon[27258]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:32.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:32 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/278362224' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:32.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:32 smithi083 ceph-mon[24755]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:32.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:32 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/278362224' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:34.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:34 smithi083 ceph-mon[24755]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:34.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:34 smithi204 ceph-mon[27258]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:36.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:36.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:36.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:36 smithi083 ceph-mon[24755]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:36.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:36.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:36.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:36 smithi204 ceph-mon[27258]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:38.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:38 smithi083 ceph-mon[24755]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:38.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:38 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:32:38.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:38 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:38.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:38 smithi204 ceph-mon[27258]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:38.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:38 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:32:38.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:38 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:39.293 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:39 smithi083 ceph-mon[24755]: Deploying daemon osd.3 on smithi083 2024-03-11T20:32:39.421 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:39 smithi204 ceph-mon[27258]: Deploying daemon osd.3 on smithi083 2024-03-11T20:32:40.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:40 smithi083 ceph-mon[24755]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:40.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:40 smithi204 ceph-mon[27258]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:42.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:42 smithi083 ceph-mon[24755]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:42.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:42 smithi204 ceph-mon[27258]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:44.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:44.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:44.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:44.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:44 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:44.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:44 smithi083 ceph-mon[24755]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:44.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:44.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:44.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:44.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:44 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:44.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:44 smithi204 ceph-mon[27258]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:46.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:46 smithi083 ceph-mon[24755]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:46.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:46 smithi083 ceph-mon[24755]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-11T20:32:46.747 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 3 on host 'smithi083' 2024-03-11T20:32:46.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:46 smithi204 ceph-mon[27258]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:46.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:46 smithi204 ceph-mon[27258]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-11T20:32:47.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:47 smithi083 ceph-mon[24755]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-11T20:32:47.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:47 smithi083 ceph-mon[24755]: osdmap e25: 4 total, 3 up, 4 in 2024-03-11T20:32:47.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:47 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:47.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:47 smithi083 ceph-mon[24755]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:32:47.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:47 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:47.712 DEBUG:teuthology.orchestra.run.smithi083:osd.3> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service 2024-03-11T20:32:47.714 INFO:tasks.cephadm:Deploying osd.4 on smithi204 with /dev/vg_nvme/lv_4... 2024-03-11T20:32:47.715 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-11T20:32:47.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:47 smithi204 ceph-mon[27258]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-11T20:32:47.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:47 smithi204 ceph-mon[27258]: osdmap e25: 4 total, 3 up, 4 in 2024-03-11T20:32:47.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:47 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:47.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:47 smithi204 ceph-mon[27258]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:32:47.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:47 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:48.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:48 smithi204 ceph-mon[27258]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:48.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:48 smithi204 ceph-mon[27258]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:32:48.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:48 smithi204 ceph-mon[27258]: osdmap e26: 4 total, 3 up, 4 in 2024-03-11T20:32:48.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:48.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:48 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:48.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:48 smithi083 ceph-mon[24755]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:48.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:48 smithi083 ceph-mon[24755]: from='osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-03-11T20:32:48.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:48 smithi083 ceph-mon[24755]: osdmap e26: 4 total, 3 up, 4 in 2024-03-11T20:32:48.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:48.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:48 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:49.689 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-11T20:32:49.689 INFO:teuthology.orchestra.run.smithi204.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-11T20:32:49.690 INFO:teuthology.orchestra.run.smithi204.stderr: stderr: 10+0 records in 2024-03-11T20:32:49.690 INFO:teuthology.orchestra.run.smithi204.stderr:10+0 records out 2024-03-11T20:32:49.690 INFO:teuthology.orchestra.run.smithi204.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0138232 s, 759 MB/s 2024-03-11T20:32:49.690 INFO:teuthology.orchestra.run.smithi204.stderr: stderr: 2024-03-11T20:32:49.690 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping successful for: 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685] boot 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: osdmap e27: 4 total, 4 up, 4 in 2024-03-11T20:32:49.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: osd.3 [v2:172.21.15.83:6826/2829470685,v1:172.21.15.83:6827/2829470685] boot 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: osdmap e27: 4 total, 4 up, 4 in 2024-03-11T20:32:49.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:32:50.296 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi204:vg_nvme/lv_4 2024-03-11T20:32:50.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:50 smithi083 ceph-mon[24755]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:50.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:50 smithi083 ceph-mon[24755]: osdmap e28: 4 total, 4 up, 4 in 2024-03-11T20:32:50.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:50 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:50.801 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:50 smithi204 ceph-mon[27258]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-11T20:32:50.801 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:50 smithi204 ceph-mon[27258]: osdmap e28: 4 total, 4 up, 4 in 2024-03-11T20:32:50.801 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:50 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:52.810 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:52 smithi083 ceph-mon[24755]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:52.810 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:52 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:32:52.811 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:52 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:32:52.811 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:52 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:52.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:52 smithi204 ceph-mon[27258]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:52.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:52 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:32:52.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:52 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:32:52.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:52 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:53.901 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:53.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:53 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:32:53.905 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:53.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:32:53.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:32:53.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:53 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:32:54.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:54 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi083 2024-03-11T20:32:54.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:54 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi083 to 3216M 2024-03-11T20:32:54.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:54 smithi083 ceph-mon[24755]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:54.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:54 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi083 2024-03-11T20:32:54.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:54 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi083 to 3216M 2024-03-11T20:32:54.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:54 smithi204 ceph-mon[27258]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:55.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/3882761939' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]: dispatch 2024-03-11T20:32:55.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]: dispatch 2024-03-11T20:32:55.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]': finished 2024-03-11T20:32:55.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: osdmap e29: 5 total, 4 up, 5 in 2024-03-11T20:32:55.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:32:55.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:55 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2679815659' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:55.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/3882761939' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]: dispatch 2024-03-11T20:32:55.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]: dispatch 2024-03-11T20:32:55.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc252c8d-49d9-45f2-844a-e34a954b91f1"}]': finished 2024-03-11T20:32:55.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: osdmap e29: 5 total, 4 up, 5 in 2024-03-11T20:32:55.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:32:55.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:55 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2679815659' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:32:56.922 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:56 smithi204 ceph-mon[27258]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:56.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:56 smithi083 ceph-mon[24755]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:58.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:58 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:58.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:58 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:58.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:32:58 smithi204 ceph-mon[27258]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:32:58.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:58 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:58.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:58 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:32:58.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:32:58 smithi083 ceph-mon[24755]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:00.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:00 smithi204 ceph-mon[27258]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:00.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:00 smithi083 ceph-mon[24755]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:01.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:33:01.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:01.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:33:01.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:02.844 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:02 smithi204 ceph-mon[27258]: Deploying daemon osd.4 on smithi204 2024-03-11T20:33:02.844 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:02 smithi204 ceph-mon[27258]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:02.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:02 smithi083 ceph-mon[24755]: Deploying daemon osd.4 on smithi204 2024-03-11T20:33:02.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:02 smithi083 ceph-mon[24755]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:04.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:04 smithi204 ceph-mon[27258]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:04.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:04 smithi083 ceph-mon[24755]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:06.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:06 smithi204 ceph-mon[27258]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:06.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:06.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:06.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:06.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:06.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:06 smithi083 ceph-mon[24755]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:06.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:06.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:06.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:06.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:08.416 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:08 smithi204 ceph-mon[27258]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:08.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:08 smithi083 ceph-mon[24755]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:09.151 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:09 smithi204 ceph-mon[27258]: from='osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:33:09.151 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:09 smithi204 ceph-mon[27258]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:33:09.225 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 4 on host 'smithi204' 2024-03-11T20:33:09.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:09 smithi083 ceph-mon[24755]: from='osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:33:09.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:09 smithi083 ceph-mon[24755]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:33:09.836 DEBUG:teuthology.orchestra.run.smithi204:osd.4> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.4.service 2024-03-11T20:33:09.839 INFO:tasks.cephadm:Deploying osd.5 on smithi204 with /dev/vg_nvme/lv_3... 2024-03-11T20:33:09.839 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-11T20:33:10.172 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-11T20:33:10.172 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: osdmap e30: 5 total, 4 up, 5 in 2024-03-11T20:33:10.172 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:10.172 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:10.173 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:10.173 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:10.173 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:10.173 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:33:10.173 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:33:10.173 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[31507]: 2024-03-11T20:33:10.058+0000 7f42bad16700 -1 osd.4 0 waiting for initial osdmap 2024-03-11T20:33:10.173 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:33:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[31507]: 2024-03-11T20:33:10.063+0000 7f42b66af700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: osdmap e30: 5 total, 4 up, 5 in 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:33:10.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:33:11.333 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:11 smithi204 ceph-mon[27258]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:11.334 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:11 smithi204 ceph-mon[27258]: osdmap e31: 5 total, 4 up, 5 in 2024-03-11T20:33:11.334 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:11.334 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:11.334 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:11.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:11 smithi083 ceph-mon[24755]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:11.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:11 smithi083 ceph-mon[24755]: osdmap e31: 5 total, 4 up, 5 in 2024-03-11T20:33:11.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:11.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:11.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:11.977 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-11T20:33:11.977 INFO:teuthology.orchestra.run.smithi204.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-11T20:33:11.977 INFO:teuthology.orchestra.run.smithi204.stderr: stderr: 10+0 records in 2024-03-11T20:33:11.977 INFO:teuthology.orchestra.run.smithi204.stderr:10+0 records out 2024-03-11T20:33:11.977 INFO:teuthology.orchestra.run.smithi204.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0120825 s, 868 MB/s 2024-03-11T20:33:11.978 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping successful for: 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423] boot 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: osdmap e32: 5 total, 5 up, 5 in 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:12.330 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:12 smithi204 ceph-mon[27258]: osdmap e33: 5 total, 5 up, 5 in 2024-03-11T20:33:12.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:33:12.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:33:12.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: osd.4 [v2:172.21.15.204:6800/190372423,v1:172.21.15.204:6801/190372423] boot 2024-03-11T20:33:12.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: osdmap e32: 5 total, 5 up, 5 in 2024-03-11T20:33:12.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:33:12.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-11T20:33:12.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:12 smithi083 ceph-mon[24755]: osdmap e33: 5 total, 5 up, 5 in 2024-03-11T20:33:12.934 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi204:vg_nvme/lv_3 2024-03-11T20:33:14.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:14 smithi204 ceph-mon[27258]: osdmap e34: 5 total, 5 up, 5 in 2024-03-11T20:33:14.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:14 smithi204 ceph-mon[27258]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:14.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:14 smithi083 ceph-mon[24755]: osdmap e34: 5 total, 5 up, 5 in 2024-03-11T20:33:14.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:14 smithi083 ceph-mon[24755]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:15.349 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:15 smithi204 ceph-mon[27258]: osdmap e35: 5 total, 5 up, 5 in 2024-03-11T20:33:15.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:15 smithi083 ceph-mon[24755]: osdmap e35: 5 total, 5 up, 5 in 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: osdmap e36: 5 total, 5 up, 5 in 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:16.252 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:16 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: osdmap e36: 5 total, 5 up, 5 in 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:16.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:16.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:16.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:16.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:16.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:16 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:17.310 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:17 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi204 2024-03-11T20:33:17.311 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:17 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi204 to 15936M 2024-03-11T20:33:17.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:17 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi204 2024-03-11T20:33:17.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:17 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi204 to 15936M 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1276582813' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]: dispatch 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]: dispatch 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]': finished 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: osdmap e37: 6 total, 5 up, 6 in 2024-03-11T20:33:18.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1276582813' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]: dispatch 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]: dispatch 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "296a2b0f-0cbb-4c83-a796-f38cb67b97b1"}]': finished 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: osdmap e37: 6 total, 5 up, 6 in 2024-03-11T20:33:18.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:19.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:19 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2781911836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:33:19.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:19 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2781911836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:33:20.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:20 smithi083 ceph-mon[24755]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:33:20.422 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:20 smithi204 ceph-mon[27258]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:33:22.864 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:22 smithi204 ceph-mon[27258]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-03-11T20:33:22.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:22 smithi083 ceph-mon[24755]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-03-11T20:33:24.809 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:24 smithi204 ceph-mon[27258]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:33:24.809 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:33:24.809 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:24.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:24 smithi083 ceph-mon[24755]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:33:24.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:33:24.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:25.672 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:25 smithi204 ceph-mon[27258]: Deploying daemon osd.5 on smithi204 2024-03-11T20:33:25.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:25 smithi083 ceph-mon[24755]: Deploying daemon osd.5 on smithi204 2024-03-11T20:33:26.856 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:26 smithi204 ceph-mon[27258]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:33:26.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:26 smithi083 ceph-mon[24755]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:33:28.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:28 smithi204 ceph-mon[27258]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:28.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:28 smithi083 ceph-mon[24755]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:30.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:30.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:30.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:30.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:30 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:30.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:30 smithi204 ceph-mon[27258]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:30 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:30.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:30 smithi083 ceph-mon[24755]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:32.471 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 5 on host 'smithi204' 2024-03-11T20:33:32.802 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:32 smithi204 ceph-mon[27258]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:32.802 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:32 smithi204 ceph-mon[27258]: from='osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:33:32.802 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:32 smithi204 ceph-mon[27258]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:33:32.802 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:32 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:32.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:32 smithi083 ceph-mon[24755]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:32.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:32 smithi083 ceph-mon[24755]: from='osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:33:32.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:32 smithi083 ceph-mon[24755]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:33:32.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:32 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:33.108 DEBUG:teuthology.orchestra.run.smithi204:osd.5> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.5.service 2024-03-11T20:33:33.110 INFO:tasks.cephadm:Deploying osd.6 on smithi204 with /dev/vg_nvme/lv_2... 2024-03-11T20:33:33.110 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-11T20:33:34.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-11T20:33:34.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: osdmap e38: 6 total, 5 up, 6 in 2024-03-11T20:33:34.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:34.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: from='osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:34.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:34.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-mon[27258]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:34.424 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[34472]: 2024-03-11T20:33:34.067+0000 7f9b149f5700 -1 osd.5 0 waiting for initial osdmap 2024-03-11T20:33:34.424 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:33:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[34472]: 2024-03-11T20:33:34.073+0000 7f9b10b8f700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: osdmap e38: 6 total, 5 up, 6 in 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: from='osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:34.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:34 smithi083 ceph-mon[24755]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-11T20:33:35.105 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:35 smithi204 ceph-mon[27258]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:35.105 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:35 smithi204 ceph-mon[27258]: osdmap e39: 6 total, 5 up, 6 in 2024-03-11T20:33:35.105 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:35.105 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:35.105 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:35 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:35.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:35 smithi083 ceph-mon[24755]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:35.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:35 smithi083 ceph-mon[24755]: osdmap e39: 6 total, 5 up, 6 in 2024-03-11T20:33:35.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:35.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:35.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:35 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:35.608 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-11T20:33:35.609 INFO:teuthology.orchestra.run.smithi204.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-11T20:33:35.609 INFO:teuthology.orchestra.run.smithi204.stderr: stderr: 10+0 records in 2024-03-11T20:33:35.609 INFO:teuthology.orchestra.run.smithi204.stderr:10+0 records out 2024-03-11T20:33:35.609 INFO:teuthology.orchestra.run.smithi204.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0122189 s, 858 MB/s 2024-03-11T20:33:35.609 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping successful for: 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404] boot 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: osdmap e40: 6 total, 6 up, 6 in 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:36.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:36 smithi204 ceph-mon[27258]: osdmap e41: 6 total, 6 up, 6 in 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: osd.5 [v2:172.21.15.204:6808/1240436404,v1:172.21.15.204:6809/1240436404] boot 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: osdmap e40: 6 total, 6 up, 6 in 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:36.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:36 smithi083 ceph-mon[24755]: osdmap e41: 6 total, 6 up, 6 in 2024-03-11T20:33:36.574 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi204:vg_nvme/lv_2 2024-03-11T20:33:38.368 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:38 smithi204 ceph-mon[27258]: osdmap e42: 6 total, 6 up, 6 in 2024-03-11T20:33:38.368 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:38 smithi204 ceph-mon[27258]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:38.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:38 smithi083 ceph-mon[24755]: osdmap e42: 6 total, 6 up, 6 in 2024-03-11T20:33:38.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:38 smithi083 ceph-mon[24755]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:39.205 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:39 smithi204 ceph-mon[27258]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:33:39.206 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:39 smithi204 ceph-mon[27258]: osdmap e43: 6 total, 6 up, 6 in 2024-03-11T20:33:39.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:39 smithi083 ceph-mon[24755]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:33:39.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:39 smithi083 ceph-mon[24755]: osdmap e43: 6 total, 6 up, 6 in 2024-03-11T20:33:40.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: osdmap e44: 6 total, 6 up, 6 in 2024-03-11T20:33:40.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:33:40.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:33:40.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:33:40.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:40.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: pgmap v137: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:40.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: osdmap e44: 6 total, 6 up, 6 in 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:40.351 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: pgmap v137: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:40.352 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi204 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi204 to 7968M 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:41.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:41.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:41.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi204 2024-03-11T20:33:41.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:41.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:33:41.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi204 to 7968M 2024-03-11T20:33:41.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:41.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:41.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:41.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:41.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:42.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:42 smithi204 ceph-mon[27258]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-11T20:33:42.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:42 smithi083 ceph-mon[24755]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-11T20:33:43.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-11T20:33:43.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: Cluster is now healthy 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2778139608' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]: dispatch 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]: dispatch 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]': finished 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: osdmap e45: 7 total, 6 up, 7 in 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:43.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:43 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/794639833' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:33:43.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-11T20:33:43.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: Cluster is now healthy 2024-03-11T20:33:43.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2778139608' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]: dispatch 2024-03-11T20:33:43.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]: dispatch 2024-03-11T20:33:43.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87c9aa45-e03a-461f-aa59-ffd7ad7a7482"}]': finished 2024-03-11T20:33:43.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: osdmap e45: 7 total, 6 up, 7 in 2024-03-11T20:33:43.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:43.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:43 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/794639833' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:33:44.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:44 smithi204 ceph-mon[27258]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:33:44.690 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:44 smithi083 ceph-mon[24755]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:33:46.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:46 smithi204 ceph-mon[27258]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-03-11T20:33:46.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:46 smithi083 ceph-mon[24755]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-03-11T20:33:48.413 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:48 smithi204 ceph-mon[27258]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:33:48.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:48 smithi083 ceph-mon[24755]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:33:49.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:33:49.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:49.346 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:33:49.346 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:50.148 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:50 smithi204 ceph-mon[27258]: Deploying daemon osd.6 on smithi204 2024-03-11T20:33:50.148 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:50 smithi204 ceph-mon[27258]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:33:50.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:50 smithi083 ceph-mon[24755]: Deploying daemon osd.6 on smithi204 2024-03-11T20:33:50.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:50 smithi083 ceph-mon[24755]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:33:52.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:52 smithi204 ceph-mon[27258]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:52.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:52 smithi083 ceph-mon[24755]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:54.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:33:54.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:56.890 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:56 smithi204 ceph-mon[27258]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:56.930 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 6 on host 'smithi204' 2024-03-11T20:33:56.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:56 smithi083 ceph-mon[24755]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:57.573 DEBUG:teuthology.orchestra.run.smithi204:osd.6> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.6.service 2024-03-11T20:33:57.576 INFO:tasks.cephadm:Deploying osd.7 on smithi204 with /dev/vg_nvme/lv_1... 2024-03-11T20:33:57.576 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-11T20:33:57.917 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:57 smithi204 ceph-mon[27258]: from='osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:33:57.917 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:57 smithi204 ceph-mon[27258]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:33:57.917 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:57 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:57.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:57 smithi083 ceph-mon[24755]: from='osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:33:57.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:57 smithi083 ceph-mon[24755]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:33:57.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:57 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:33:58.811 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:58.811 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-11T20:33:58.811 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: osdmap e46: 7 total, 6 up, 7 in 2024-03-11T20:33:58.811 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:58.812 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: from='osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:58.812 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-mon[27258]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:58.812 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[37436]: 2024-03-11T20:33:58.624+0000 7f30a1cf7700 -1 osd.6 0 waiting for initial osdmap 2024-03-11T20:33:58.812 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:33:58 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[37436]: 2024-03-11T20:33:58.633+0000 7f309c68e700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:33:58.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:33:58.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-11T20:33:58.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: osdmap e46: 7 total, 6 up, 7 in 2024-03-11T20:33:58.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:58.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: from='osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:58.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:58 smithi083 ceph-mon[24755]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:33:59.711 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:59 smithi204 ceph-mon[27258]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:59.711 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:59 smithi204 ceph-mon[27258]: osdmap e47: 7 total, 6 up, 7 in 2024-03-11T20:33:59.711 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:59.711 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:33:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:59.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:59 smithi083 ceph-mon[24755]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:33:59.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:59 smithi083 ceph-mon[24755]: osdmap e47: 7 total, 6 up, 7 in 2024-03-11T20:33:59.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:33:59.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:33:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:34:00.221 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-11T20:34:00.222 INFO:teuthology.orchestra.run.smithi204.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-11T20:34:00.222 INFO:teuthology.orchestra.run.smithi204.stderr: stderr: 10+0 records in 2024-03-11T20:34:00.222 INFO:teuthology.orchestra.run.smithi204.stderr:10+0 records out 2024-03-11T20:34:00.222 INFO:teuthology.orchestra.run.smithi204.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125986 s, 832 MB/s 2024-03-11T20:34:00.222 INFO:teuthology.orchestra.run.smithi204.stderr:--> Zapping successful for: 2024-03-11T20:34:00.822 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244] boot 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: osdmap e48: 7 total, 7 up, 7 in 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:00.823 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:00 smithi204 ceph-mon[27258]: osdmap e49: 7 total, 7 up, 7 in 2024-03-11T20:34:00.845 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch daemon add osd smithi204:vg_nvme/lv_1 2024-03-11T20:34:00.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: osd.6 [v2:172.21.15.204:6816/1429275244,v1:172.21.15.204:6817/1429275244] boot 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: osdmap e48: 7 total, 7 up, 7 in 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:00.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:00 smithi083 ceph-mon[24755]: osdmap e49: 7 total, 7 up, 7 in 2024-03-11T20:34:02.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:02 smithi204 ceph-mon[27258]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:02.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:02 smithi204 ceph-mon[27258]: osdmap e50: 7 total, 7 up, 7 in 2024-03-11T20:34:02.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:02 smithi083 ceph-mon[24755]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:02.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:02 smithi083 ceph-mon[24755]: osdmap e50: 7 total, 7 up, 7 in 2024-03-11T20:34:04.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:04 smithi204 ceph-mon[27258]: osdmap e51: 7 total, 7 up, 7 in 2024-03-11T20:34:04.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:04 smithi204 ceph-mon[27258]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:04.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:04 smithi083 ceph-mon[24755]: osdmap e51: 7 total, 7 up, 7 in 2024-03-11T20:34:04.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:04 smithi083 ceph-mon[24755]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:05.308 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:05 smithi204 ceph-mon[27258]: osdmap e52: 7 total, 7 up, 7 in 2024-03-11T20:34:05.308 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:05 smithi204 ceph-mon[27258]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:34:05.309 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:34:05.309 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:34:05.309 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:05.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:05 smithi083 ceph-mon[24755]: osdmap e52: 7 total, 7 up, 7 in 2024-03-11T20:34:05.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:05 smithi083 ceph-mon[24755]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi204:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:34:05.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-11T20:34:05.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-11T20:34:05.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:06.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi204 2024-03-11T20:34:06.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi204 to 5312M 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:06.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:06 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi204 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi204 to 5312M 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:06.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:06.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:06.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:06 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:08.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/2501444565' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]: dispatch 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]: dispatch 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]': finished 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: osdmap e53: 8 total, 7 up, 8 in 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-11T20:34:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:08 smithi204 ceph-mon[27258]: from='client.? 172.21.15.204:0/1587987788' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/2501444565' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]: dispatch 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]: dispatch 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5634b258-1d78-4795-84f2-40ad4d1e41c3"}]': finished 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: osdmap e53: 8 total, 7 up, 8 in 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-03-11T20:34:08.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:08 smithi083 ceph-mon[24755]: from='client.? 172.21.15.204:0/1587987788' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-11T20:34:09.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:34:09.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:34:09.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:34:09.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:34:10.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:10 smithi204 ceph-mon[27258]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-03-11T20:34:10.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:10 smithi083 ceph-mon[24755]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-03-11T20:34:12.673 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:12 smithi204 ceph-mon[27258]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-11T20:34:12.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:12 smithi083 ceph-mon[24755]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-11T20:34:13.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:13 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:34:13.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:13 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:13.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:34:13.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:14.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:14 smithi204 ceph-mon[27258]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-03-11T20:34:14.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:14 smithi204 ceph-mon[27258]: Deploying daemon osd.7 on smithi204 2024-03-11T20:34:14.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:14 smithi083 ceph-mon[24755]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-03-11T20:34:14.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:14 smithi083 ceph-mon[24755]: Deploying daemon osd.7 on smithi204 2024-03-11T20:34:16.844 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:16 smithi204 ceph-mon[27258]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:34:16.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:16 smithi083 ceph-mon[24755]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:34:18.126 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:18 smithi204 ceph-mon[27258]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:18.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:18 smithi083 ceph-mon[24755]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:19.996 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:19.997 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:19.997 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:19.997 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:20.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:20.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:20.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:20.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:20.861 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:20 smithi204 ceph-mon[27258]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:21.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:20 smithi083 ceph-mon[24755]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:21.544 INFO:teuthology.orchestra.run.smithi204.stdout:Created osd(s) 7 on host 'smithi204' 2024-03-11T20:34:22.466 DEBUG:teuthology.orchestra.run.smithi204:osd.7> sudo journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.7.service 2024-03-11T20:34:22.468 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-11T20:34:22.469 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd stat -f json 2024-03-11T20:34:22.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:22.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:22 smithi083 ceph-mon[24755]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:22.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:22 smithi083 ceph-mon[24755]: from='osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:34:22.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:22 smithi083 ceph-mon[24755]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:34:22.813 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:22 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:22.813 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:22 smithi204 ceph-mon[27258]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:22.813 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:22 smithi204 ceph-mon[27258]: from='osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:34:22.814 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:22 smithi204 ceph-mon[27258]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: osdmap e54: 8 total, 7 up, 8 in 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: from='osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:34:23.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:23.674 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[40505]: 2024-03-11T20:34:23.572+0000 7f6b51aec700 -1 osd.7 0 waiting for initial osdmap 2024-03-11T20:34:23.674 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:34:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[40505]: 2024-03-11T20:34:23.582+0000 7f6b4c483700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:34:23.915 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-11T20:34:23.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: osdmap e54: 8 total, 7 up, 8 in 2024-03-11T20:34:23.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:23.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: from='osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:34:23.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:34:23.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:23 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:24.688 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:34:24.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:24.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: osdmap e55: 8 total, 7 up, 8 in 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717] boot 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: osdmap e56: 8 total, 8 up, 8 in 2024-03-11T20:34:24.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:24 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:24.846 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-11T20:34:24.846 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]': finished 2024-03-11T20:34:24.846 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: osdmap e55: 8 total, 7 up, 8 in 2024-03-11T20:34:24.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:24.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:24.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: osd.7 [v2:172.21.15.204:6824/3834660717,v1:172.21.15.204:6825/3834660717] boot 2024-03-11T20:34:24.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: osdmap e56: 8 total, 8 up, 8 in 2024-03-11T20:34:24.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:24 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:34:25.391 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1710189264,"num_in_osds":8,"osd_in_since":1710189247,"num_remapped_pgs":1} 2024-03-11T20:34:25.391 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd dump --format=json 2024-03-11T20:34:25.625 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:25.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:25 smithi083 ceph-mon[24755]: purged_snaps scrub starts 2024-03-11T20:34:25.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:25 smithi083 ceph-mon[24755]: purged_snaps scrub ok 2024-03-11T20:34:25.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:25 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/603144267' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-11T20:34:25.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:25 smithi083 ceph-mon[24755]: osdmap e57: 8 total, 8 up, 8 in 2024-03-11T20:34:25.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:25 smithi204 ceph-mon[27258]: purged_snaps scrub starts 2024-03-11T20:34:25.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:25 smithi204 ceph-mon[27258]: purged_snaps scrub ok 2024-03-11T20:34:25.847 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:25 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/603144267' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-11T20:34:25.848 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:25 smithi204 ceph-mon[27258]: osdmap e57: 8 total, 8 up, 8 in 2024-03-11T20:34:26.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:26 smithi083 ceph-mon[24755]: osdmap e58: 8 total, 8 up, 8 in 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:34:26.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:34:26.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:34:26.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:34:26.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:26 smithi204 ceph-mon[27258]: osdmap e58: 8 total, 8 up, 8 in 2024-03-11T20:34:27.352 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:34:27.352 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":59,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","created":"2024-03-11T20:28:17.230849+0000","modified":"2024-03-11T20:34:27.057368+0000","last_up_change":"2024-03-11T20:34:24.562405+0000","last_in_change":"2024-03-11T20:34:07.208276+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-11T20:32:21.830712+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d63f4854-b6ea-43a1-8f55-a345f136033d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6803","nonce":557961932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6805","nonce":557961932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6809","nonce":557961932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6807","nonce":557961932}]},"public_addr":"172.21.15.83:6803/557961932","cluster_addr":"172.21.15.83:6805/557961932","heartbeat_back_addr":"172.21.15.83:6809/557961932","heartbeat_front_addr":"172.21.15.83:6807/557961932","state":["exists","up"]},{"osd":1,"uuid":"75c3c297-0eb7-4fd4-902a-3e69959edd14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6811","nonce":2904385135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6813","nonce":2904385135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6817","nonce":2904385135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6815","nonce":2904385135}]},"public_addr":"172.21.15.83:6811/2904385135","cluster_addr":"172.21.15.83:6813/2904385135","heartbeat_back_addr":"172.21.15.83:6817/2904385135","heartbeat_front_addr":"172.21.15.83:6815/2904385135","state":["exists","up"]},{"osd":2,"uuid":"f9b2be69-985a-4bc9-98de-17219ec4bb04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6819","nonce":882489284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6821","nonce":882489284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6825","nonce":882489284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6823","nonce":882489284}]},"public_addr":"172.21.15.83:6819/882489284","cluster_addr":"172.21.15.83:6821/882489284","heartbeat_back_addr":"172.21.15.83:6825/882489284","heartbeat_front_addr":"172.21.15.83:6823/882489284","state":["exists","up"]},{"osd":3,"uuid":"93517251-5f44-4d7b-8624-4b4c3f0a9f4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6827","nonce":2829470685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6829","nonce":2829470685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6833","nonce":2829470685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6831","nonce":2829470685}]},"public_addr":"172.21.15.83:6827/2829470685","cluster_addr":"172.21.15.83:6829/2829470685","heartbeat_back_addr":"172.21.15.83:6833/2829470685","heartbeat_front_addr":"172.21.15.83:6831/2829470685","state":["exists","up"]},{"osd":4,"uuid":"bc252c8d-49d9-45f2-844a-e34a954b91f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6801","nonce":190372423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6803","nonce":190372423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6807","nonce":190372423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6805","nonce":190372423}]},"public_addr":"172.21.15.204:6801/190372423","cluster_addr":"172.21.15.204:6803/190372423","heartbeat_back_addr":"172.21.15.204:6807/190372423","heartbeat_front_addr":"172.21.15.204:6805/190372423","state":["exists","up"]},{"osd":5,"uuid":"296a2b0f-0cbb-4c83-a796-f38cb67b97b1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6809","nonce":1240436404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6811","nonce":1240436404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6815","nonce":1240436404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6813","nonce":1240436404}]},"public_addr":"172.21.15.204:6809/1240436404","cluster_addr":"172.21.15.204:6811/1240436404","heartbeat_back_addr":"172.21.15.204:6815/1240436404","heartbeat_front_addr":"172.21.15.204:6813/1240436404","state":["exists","up"]},{"osd":6,"uuid":"87c9aa45-e03a-461f-aa59-ffd7ad7a7482","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6817","nonce":1429275244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6819","nonce":1429275244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6823","nonce":1429275244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6821","nonce":1429275244}]},"public_addr":"172.21.15.204:6817/1429275244","cluster_addr":"172.21.15.204:6819/1429275244","heartbeat_back_addr":"172.21.15.204:6823/1429275244","heartbeat_front_addr":"172.21.15.204:6821/1429275244","state":["exists","up"]},{"osd":7,"uuid":"5634b258-1d78-4795-84f2-40ad4d1e41c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6825","nonce":3834660717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6827","nonce":3834660717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6831","nonce":3834660717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6829","nonce":3834660717}]},"public_addr":"172.21.15.204:6825/3834660717","cluster_addr":"172.21.15.204:6827/3834660717","heartbeat_back_addr":"172.21.15.204:6831/3834660717","heartbeat_front_addr":"172.21.15.204:6829/3834660717","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:17.587866+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:48.623522+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:18.716641+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:47.410587+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:09.802096+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:33.422403+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:57.950529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:34:22.840790+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6800/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/1537548594":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/903273449":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/10611879":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2263980434":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6801/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:6800/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2897985633":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6800/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/3439912351":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6801/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/4001148915":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/34911146":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/1115708401":"2024-03-12T20:28:41.474101+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-11T20:34:27.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:27 smithi083 ceph-mon[24755]: Detected new or changed devices on smithi204 2024-03-11T20:34:27.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:27 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi204 to 3984M 2024-03-11T20:34:27.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:27 smithi083 ceph-mon[24755]: osdmap e59: 8 total, 8 up, 8 in 2024-03-11T20:34:27.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:27 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2883041167' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:27 smithi204 ceph-mon[27258]: Detected new or changed devices on smithi204 2024-03-11T20:34:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:27 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi204 to 3984M 2024-03-11T20:34:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:27 smithi204 ceph-mon[27258]: osdmap e59: 8 total, 8 up, 8 in 2024-03-11T20:34:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:27 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2883041167' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:28.118 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-11T20:32:21.830712+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-11T20:34:28.118 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-11T20:34:28.337 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:28.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:28 smithi083 ceph-mon[24755]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:28.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:28 smithi083 ceph-mon[24755]: osdmap e60: 8 total, 8 up, 8 in 2024-03-11T20:34:28.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:28 smithi204 ceph-mon[27258]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:28.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:28 smithi204 ceph-mon[27258]: osdmap e60: 8 total, 8 up, 8 in 2024-03-11T20:34:30.331 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2024-03-11T20:34:30.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:30 smithi083 ceph-mon[24755]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:30.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:30 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/459909247' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-11T20:34:30.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:30 smithi204 ceph-mon[27258]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:30.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:30 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/459909247' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-11T20:34:30.973 INFO:tasks.cephadm:Setting up client nodes... 2024-03-11T20:34:30.973 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-11T20:34:32.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:32 smithi083 ceph-mon[24755]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:34:32.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:32 smithi204 ceph-mon[27258]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-11T20:34:33.116 INFO:teuthology.orchestra.run.smithi083.stdout:[client.0] 2024-03-11T20:34:33.116 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQDZau9l9cBiBxAAfrspIcgYcrgwRRlZWpqJRw== 2024-03-11T20:34:33.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:33 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/768832244' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-11T20:34:33.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:33 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/768832244' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-11T20:34:33.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/768832244' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-11T20:34:33.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:33 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/768832244' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-11T20:34:33.955 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-03-11T20:34:33.955 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-11T20:34:33.955 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-11T20:34:33.996 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-11T20:34:33.997 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-11T20:34:33.997 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph mgr dump --format=json 2024-03-11T20:34:34.249 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:34.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:34 smithi083 ceph-mon[24755]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-03-11T20:34:34.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:34 smithi204 ceph-mon[27258]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-03-11T20:34:36.186 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:34:36.912 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:36 smithi083 ceph-mon[24755]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:34:36.912 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:36 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2109381423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-11T20:34:36.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:36 smithi204 ceph-mon[27258]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-11T20:34:36.923 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:36 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2109381423' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-11T20:34:36.935 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":19,"active_gid":14206,"active_name":"smithi083.jyxogi","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6800","nonce":32265062},{"type":"v1","addr":"172.21.15.83:6801","nonce":32265062}]},"active_addr":"172.21.15.83:6801/32265062","active_change":"2024-03-11T20:30:09.461675+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14226,"name":"smithi204.qknpir","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.83:8443/","prometheus":"http://172.21.15.83:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3591217828}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4062097169}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":606382970}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":593763020}]}]}} 2024-03-11T20:34:36.940 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-11T20:34:36.940 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-11T20:34:36.941 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd dump --format=json 2024-03-11T20:34:37.157 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:38.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:38 smithi083 ceph-mon[24755]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:34:38.423 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:38 smithi204 ceph-mon[27258]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-11T20:34:39.066 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:34:39.067 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","created":"2024-03-11T20:28:17.230849+0000","modified":"2024-03-11T20:34:28.064943+0000","last_up_change":"2024-03-11T20:34:24.562405+0000","last_in_change":"2024-03-11T20:34:07.208276+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-11T20:32:21.830712+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d63f4854-b6ea-43a1-8f55-a345f136033d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6803","nonce":557961932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6805","nonce":557961932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6809","nonce":557961932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6807","nonce":557961932}]},"public_addr":"172.21.15.83:6803/557961932","cluster_addr":"172.21.15.83:6805/557961932","heartbeat_back_addr":"172.21.15.83:6809/557961932","heartbeat_front_addr":"172.21.15.83:6807/557961932","state":["exists","up"]},{"osd":1,"uuid":"75c3c297-0eb7-4fd4-902a-3e69959edd14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6811","nonce":2904385135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6813","nonce":2904385135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6817","nonce":2904385135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6815","nonce":2904385135}]},"public_addr":"172.21.15.83:6811/2904385135","cluster_addr":"172.21.15.83:6813/2904385135","heartbeat_back_addr":"172.21.15.83:6817/2904385135","heartbeat_front_addr":"172.21.15.83:6815/2904385135","state":["exists","up"]},{"osd":2,"uuid":"f9b2be69-985a-4bc9-98de-17219ec4bb04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6819","nonce":882489284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6821","nonce":882489284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6825","nonce":882489284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6823","nonce":882489284}]},"public_addr":"172.21.15.83:6819/882489284","cluster_addr":"172.21.15.83:6821/882489284","heartbeat_back_addr":"172.21.15.83:6825/882489284","heartbeat_front_addr":"172.21.15.83:6823/882489284","state":["exists","up"]},{"osd":3,"uuid":"93517251-5f44-4d7b-8624-4b4c3f0a9f4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6827","nonce":2829470685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6829","nonce":2829470685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6833","nonce":2829470685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6831","nonce":2829470685}]},"public_addr":"172.21.15.83:6827/2829470685","cluster_addr":"172.21.15.83:6829/2829470685","heartbeat_back_addr":"172.21.15.83:6833/2829470685","heartbeat_front_addr":"172.21.15.83:6831/2829470685","state":["exists","up"]},{"osd":4,"uuid":"bc252c8d-49d9-45f2-844a-e34a954b91f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6801","nonce":190372423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6803","nonce":190372423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6807","nonce":190372423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6805","nonce":190372423}]},"public_addr":"172.21.15.204:6801/190372423","cluster_addr":"172.21.15.204:6803/190372423","heartbeat_back_addr":"172.21.15.204:6807/190372423","heartbeat_front_addr":"172.21.15.204:6805/190372423","state":["exists","up"]},{"osd":5,"uuid":"296a2b0f-0cbb-4c83-a796-f38cb67b97b1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6809","nonce":1240436404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6811","nonce":1240436404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6815","nonce":1240436404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6813","nonce":1240436404}]},"public_addr":"172.21.15.204:6809/1240436404","cluster_addr":"172.21.15.204:6811/1240436404","heartbeat_back_addr":"172.21.15.204:6815/1240436404","heartbeat_front_addr":"172.21.15.204:6813/1240436404","state":["exists","up"]},{"osd":6,"uuid":"87c9aa45-e03a-461f-aa59-ffd7ad7a7482","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6817","nonce":1429275244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6819","nonce":1429275244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6823","nonce":1429275244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6821","nonce":1429275244}]},"public_addr":"172.21.15.204:6817/1429275244","cluster_addr":"172.21.15.204:6819/1429275244","heartbeat_back_addr":"172.21.15.204:6823/1429275244","heartbeat_front_addr":"172.21.15.204:6821/1429275244","state":["exists","up"]},{"osd":7,"uuid":"5634b258-1d78-4795-84f2-40ad4d1e41c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6825","nonce":3834660717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6827","nonce":3834660717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6831","nonce":3834660717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6829","nonce":3834660717}]},"public_addr":"172.21.15.204:6825/3834660717","cluster_addr":"172.21.15.204:6827/3834660717","heartbeat_back_addr":"172.21.15.204:6831/3834660717","heartbeat_front_addr":"172.21.15.204:6829/3834660717","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:17.587866+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:48.623522+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:18.716641+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:47.410587+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:09.802096+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:33.422403+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:57.950529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:34:22.840790+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6800/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/1537548594":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/903273449":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/10611879":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2263980434":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6801/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:6800/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2897985633":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6800/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/3439912351":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6801/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/4001148915":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/34911146":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/1115708401":"2024-03-12T20:28:41.474101+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-11T20:34:39.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:39 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1399487356' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:39.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:39 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1399487356' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:39.777 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-11T20:34:39.778 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd dump --format=json 2024-03-11T20:34:39.992 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:40.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:40 smithi204 ceph-mon[27258]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-03-11T20:34:40.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:40 smithi083 ceph-mon[24755]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-03-11T20:34:41.973 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:34:41.973 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"c3f853a8-dfe5-11ee-95c8-87774f69a715","created":"2024-03-11T20:28:17.230849+0000","modified":"2024-03-11T20:34:28.064943+0000","last_up_change":"2024-03-11T20:34:24.562405+0000","last_in_change":"2024-03-11T20:34:07.208276+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-11T20:32:21.830712+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d63f4854-b6ea-43a1-8f55-a345f136033d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6803","nonce":557961932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6805","nonce":557961932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6809","nonce":557961932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":557961932},{"type":"v1","addr":"172.21.15.83:6807","nonce":557961932}]},"public_addr":"172.21.15.83:6803/557961932","cluster_addr":"172.21.15.83:6805/557961932","heartbeat_back_addr":"172.21.15.83:6809/557961932","heartbeat_front_addr":"172.21.15.83:6807/557961932","state":["exists","up"]},{"osd":1,"uuid":"75c3c297-0eb7-4fd4-902a-3e69959edd14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6811","nonce":2904385135}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6813","nonce":2904385135}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6817","nonce":2904385135}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":2904385135},{"type":"v1","addr":"172.21.15.83:6815","nonce":2904385135}]},"public_addr":"172.21.15.83:6811/2904385135","cluster_addr":"172.21.15.83:6813/2904385135","heartbeat_back_addr":"172.21.15.83:6817/2904385135","heartbeat_front_addr":"172.21.15.83:6815/2904385135","state":["exists","up"]},{"osd":2,"uuid":"f9b2be69-985a-4bc9-98de-17219ec4bb04","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6819","nonce":882489284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6821","nonce":882489284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6825","nonce":882489284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":882489284},{"type":"v1","addr":"172.21.15.83:6823","nonce":882489284}]},"public_addr":"172.21.15.83:6819/882489284","cluster_addr":"172.21.15.83:6821/882489284","heartbeat_back_addr":"172.21.15.83:6825/882489284","heartbeat_front_addr":"172.21.15.83:6823/882489284","state":["exists","up"]},{"osd":3,"uuid":"93517251-5f44-4d7b-8624-4b4c3f0a9f4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6827","nonce":2829470685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6829","nonce":2829470685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6833","nonce":2829470685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2829470685},{"type":"v1","addr":"172.21.15.83:6831","nonce":2829470685}]},"public_addr":"172.21.15.83:6827/2829470685","cluster_addr":"172.21.15.83:6829/2829470685","heartbeat_back_addr":"172.21.15.83:6833/2829470685","heartbeat_front_addr":"172.21.15.83:6831/2829470685","state":["exists","up"]},{"osd":4,"uuid":"bc252c8d-49d9-45f2-844a-e34a954b91f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6800","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6801","nonce":190372423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6802","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6803","nonce":190372423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6806","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6807","nonce":190372423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6804","nonce":190372423},{"type":"v1","addr":"172.21.15.204:6805","nonce":190372423}]},"public_addr":"172.21.15.204:6801/190372423","cluster_addr":"172.21.15.204:6803/190372423","heartbeat_back_addr":"172.21.15.204:6807/190372423","heartbeat_front_addr":"172.21.15.204:6805/190372423","state":["exists","up"]},{"osd":5,"uuid":"296a2b0f-0cbb-4c83-a796-f38cb67b97b1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6808","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6809","nonce":1240436404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6810","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6811","nonce":1240436404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6814","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6815","nonce":1240436404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6812","nonce":1240436404},{"type":"v1","addr":"172.21.15.204:6813","nonce":1240436404}]},"public_addr":"172.21.15.204:6809/1240436404","cluster_addr":"172.21.15.204:6811/1240436404","heartbeat_back_addr":"172.21.15.204:6815/1240436404","heartbeat_front_addr":"172.21.15.204:6813/1240436404","state":["exists","up"]},{"osd":6,"uuid":"87c9aa45-e03a-461f-aa59-ffd7ad7a7482","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6816","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6817","nonce":1429275244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6818","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6819","nonce":1429275244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6822","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6823","nonce":1429275244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6820","nonce":1429275244},{"type":"v1","addr":"172.21.15.204:6821","nonce":1429275244}]},"public_addr":"172.21.15.204:6817/1429275244","cluster_addr":"172.21.15.204:6819/1429275244","heartbeat_back_addr":"172.21.15.204:6823/1429275244","heartbeat_front_addr":"172.21.15.204:6821/1429275244","state":["exists","up"]},{"osd":7,"uuid":"5634b258-1d78-4795-84f2-40ad4d1e41c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6824","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6825","nonce":3834660717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6826","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6827","nonce":3834660717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6830","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6831","nonce":3834660717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.204:6828","nonce":3834660717},{"type":"v1","addr":"172.21.15.204:6829","nonce":3834660717}]},"public_addr":"172.21.15.204:6825/3834660717","cluster_addr":"172.21.15.204:6827/3834660717","heartbeat_back_addr":"172.21.15.204:6831/3834660717","heartbeat_front_addr":"172.21.15.204:6829/3834660717","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:17.587866+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:31:48.623522+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:18.716641+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:32:47.410587+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:09.802096+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:33.422403+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:33:57.950529+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-11T20:34:22.840790+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6801/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6800/1794506621":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/1537548594":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/903273449":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/10611879":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2263980434":"2024-03-12T20:30:09.461396+0000","172.21.15.83:6801/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:6800/2762807569":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/2897985633":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6800/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/3439912351":"2024-03-12T20:28:41.474101+0000","172.21.15.83:6801/3939418844":"2024-03-12T20:28:41.474101+0000","172.21.15.83:0/4001148915":"2024-03-12T20:30:09.461396+0000","172.21.15.83:0/34911146":"2024-03-12T20:29:11.285306+0000","172.21.15.83:0/1115708401":"2024-03-12T20:28:41.474101+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-11T20:34:42.584 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-11T20:34:42.585 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-11T20:34:42.585 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-11T20:34:42.586 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-11T20:34:42.586 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-11T20:34:42.586 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-11T20:34:42.587 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-11T20:34:42.587 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-11T20:34:42.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:42 smithi083 ceph-mon[24755]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-03-11T20:34:42.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:42 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2716850827' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:42.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:42 smithi204 ceph-mon[27258]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-03-11T20:34:42.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:42 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2716850827' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-11T20:34:43.026 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.027 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.027 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.028 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.035 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.036 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.036 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:43.039 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:44.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:44 smithi204 ceph-mon[27258]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:44.929 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:44 smithi083 ceph-mon[24755]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:46.659 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:46 smithi083 ceph-mon[24755]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:46.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:46 smithi204 ceph-mon[27258]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:48.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:48 smithi204 ceph-mon[27258]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:48.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:48 smithi083 ceph-mon[24755]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:50.874 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:50 smithi083 ceph-mon[24755]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:50.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:50 smithi204 ceph-mon[27258]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:52.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:52 smithi204 ceph-mon[27258]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:52.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:52 smithi083 ceph-mon[24755]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:52.976 INFO:teuthology.orchestra.run.smithi083.stdout:81604378657 2024-03-11T20:34:52.976 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-11T20:34:53.553 INFO:teuthology.orchestra.run.smithi083.stdout:137438953496 2024-03-11T20:34:53.553 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-11T20:34:54.331 INFO:teuthology.orchestra.run.smithi083.stdout:171798691858 2024-03-11T20:34:54.332 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-11T20:34:54.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:54 smithi204 ceph-mon[27258]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:54.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:54 smithi083 ceph-mon[24755]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:56.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:56 smithi204 ceph-mon[27258]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:56.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:56 smithi083 ceph-mon[24755]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:57.140 INFO:teuthology.orchestra.run.smithi083.stdout:60129542184 2024-03-11T20:34:57.140 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-11T20:34:57.655 INFO:teuthology.orchestra.run.smithi083.stdout:115964117021 2024-03-11T20:34:57.655 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-11T20:34:58.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:34:58 smithi083 ceph-mon[24755]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:58.191 INFO:teuthology.orchestra.run.smithi083.stdout:240518168584 2024-03-11T20:34:58.191 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-11T20:34:58.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:34:58 smithi204 ceph-mon[27258]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:34:58.638 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:58.639 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:58.642 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:58.646 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:34:59.039 INFO:teuthology.orchestra.run.smithi083.stdout:38654705711 2024-03-11T20:34:59.039 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-11T20:35:00.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:00 smithi083 ceph-mon[24755]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:00.766 INFO:teuthology.orchestra.run.smithi083.stdout:81604378659 2024-03-11T20:35:00.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:00 smithi204 ceph-mon[27258]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:01.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:01 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2062729563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-11T20:35:01.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:01 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2062729563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-11T20:35:02.919 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:02 smithi083 ceph-mon[24755]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:02.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:02 smithi204 ceph-mon[27258]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:03.229 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:03.229 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:03.229 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:03.687 INFO:teuthology.orchestra.run.smithi083.stdout:206158430223 2024-03-11T20:35:03.688 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-11T20:35:04.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:04 smithi204 ceph-mon[27258]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:04.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:04 smithi083 ceph-mon[24755]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:05.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:05 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/30103636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-11T20:35:05.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:05 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/30103636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-11T20:35:06.731 INFO:teuthology.orchestra.run.smithi083.stdout:115964117022 2024-03-11T20:35:06.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:06 smithi204 ceph-mon[27258]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:06.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:06 smithi083 ceph-mon[24755]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:07.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117021 got 115964117022 for osd.3 2024-03-11T20:35:07.234 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:08 smithi083 ceph-mon[24755]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:08.408 INFO:teuthology.orchestra.run.smithi083.stdout:240518168587 2024-03-11T20:35:08.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:08 smithi204 ceph-mon[27258]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:09.179 INFO:teuthology.orchestra.run.smithi083.stdout:171798691861 2024-03-11T20:35:09.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:09 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1133845740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-11T20:35:09.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:09 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4182819742' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-11T20:35:09.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:09 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1133845740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-11T20:35:09.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:09 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4182819742' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-11T20:35:09.739 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691858 got 171798691861 for osd.5 2024-03-11T20:35:09.739 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:10.116 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378659 for osd.2 2024-03-11T20:35:10.117 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:10.320 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:10 smithi083 ceph-mon[24755]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:10.320 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:35:10.320 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:35:10.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:10 smithi204 ceph-mon[27258]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:10.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:35:10.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:35:10.727 INFO:teuthology.orchestra.run.smithi083.stdout:38654705713 2024-03-11T20:35:11.083 INFO:teuthology.orchestra.run.smithi083.stdout:60129542187 2024-03-11T20:35:11.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:11 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1661812059' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-11T20:35:11.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:11 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1661812059' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-11T20:35:11.672 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-03-11T20:35:11.672 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:12.115 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:12.143 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168584 got 240518168587 for osd.7 2024-03-11T20:35:12.143 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:12.389 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:12 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2273020202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-11T20:35:12.389 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:12 smithi083 ceph-mon[24755]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:12.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:12 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2273020202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-11T20:35:12.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:12 smithi204 ceph-mon[27258]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:14.597 INFO:teuthology.orchestra.run.smithi083.stdout:137438953500 2024-03-11T20:35:14.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705711 got 38654705713 for osd.0 2024-03-11T20:35:14.724 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:14.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:14 smithi204 ceph-mon[27258]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:14.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:14 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2887417736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-11T20:35:14.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:14 smithi083 ceph-mon[24755]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:14.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:14 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2887417736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-11T20:35:15.498 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953500 for osd.4 2024-03-11T20:35:15.499 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:15.883 INFO:teuthology.orchestra.run.smithi083.stdout:206158430226 2024-03-11T20:35:16.506 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430226 for osd.6 2024-03-11T20:35:16.506 DEBUG:teuthology.parallel:result is None 2024-03-11T20:35:16.507 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-11T20:35:16.507 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph pg dump --format=json 2024-03-11T20:35:16.721 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:16.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:16 smithi083 ceph-mon[24755]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:16.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:16 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4019272735' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-11T20:35:16.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:16 smithi204 ceph-mon[27258]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:16.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:16 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4019272735' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-11T20:35:18.327 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:18 smithi083 ceph-mon[24755]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:18.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:18 smithi204 ceph-mon[27258]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:18.535 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:18.537 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-03-11T20:35:19.138 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-03-11T20:35:17.568495+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48992,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912224,"statfs":{"total":767960285184,"available":767910117376,"internally_reserved":0,"allocated":6520832,"data_stored":3571156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002325"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-11T20:34:28.075116+0000","last_change":"2024-03-11T20:34:28.075116+0000","last_active":"2024-03-11T20:34:28.075116+0000","last_peered":"2024-03-11T20:34:28.075116+0000","last_clean":"2024-03-11T20:34:28.075116+0000","last_became_active":"2024-03-11T20:34:28.074700+0000","last_became_peered":"2024-03-11T20:34:28.074700+0000","last_unstale":"2024-03-11T20:34:28.075116+0000","last_undegraded":"2024-03-11T20:34:28.075116+0000","last_fullsized":"2024-03-11T20:34:28.075116+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-11T20:32:22.052597+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-11T20:32:22.052597+0000","last_clean_scrub_stamp":"2024-03-11T20:32:22.052597+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-13T03:32:21.620934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67900000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.502}]},{"osd":2,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.38},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.433}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.32800000000000001,"15min":0.31},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.47999999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.32800000000000001},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.54500000000000004,"5min":0.63,"15min":0.63},"last":0.377}]},{"osd":2,"last update":"Mon Mar 11 20:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.39400000000000002,"15min":0.38800000000000001},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Mar 11 20:34:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.434,"15min":0.436},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58499999999999996,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.434,"5min":0.46300000000000002,"15min":0.46800000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.55100000000000005}]},{"osd":4,"last update":"Mon Mar 11 20:35:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.47499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.442,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Mar 11 20:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.73999999999999999}]},{"osd":6,"last update":"Mon Mar 11 20:35:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.77000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Mar 11 20:34:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.315},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.33600000000000002,"15min":0.308},"min":{"1min":0.27500000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.433}]},{"osd":2,"last update":"Mon Mar 11 20:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.34200000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.40300000000000002,"15min":0.39500000000000002},"min":{"1min":0.33200000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.42599999999999999}]},{"osd":3,"last update":"Mon Mar 11 20:34:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45700000000000002,"15min":0.45800000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.29699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Mar 11 20:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.62}]},{"osd":5,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.59999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6304,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738848,"statfs":{"total":95995035648,"available":95988580352,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36599999999999999},"min":{"1min":0.25900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36599999999999999},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Mar 11 20:34:24 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.246},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.38500000000000001,"15min":0.379},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.307}]},{"osd":3,"last update":"Mon Mar 11 20:34:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40999999999999998,"15min":0.41199999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57199999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.316},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.372}]},{"osd":4,"last update":"Mon Mar 11 20:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81399999999999995,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58899999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.74099999999999999}]},{"osd":5,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.66400000000000003}]},{"osd":6,"last update":"Mon Mar 11 20:35:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.622},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.78800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75900000000000001}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41599999999999998,"15min":0.41499999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.41699999999999998,"15min":0.41099999999999998},"min":{"1min":0.307,"5min":0.246,"15min":0.246},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.374},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":0.308}]},{"osd":2,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45600000000000002,"15min":0.45400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.436,"15min":0.434},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Mar 11 20:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Mar 11 20:34:37 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Mar 11 20:35:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.746}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":1.143,"15min":1.143},"last":0.628},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51800000000000002}]},{"osd":1,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.35499999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73899999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58199999999999996,"15min":0.57699999999999996},"min":{"1min":0.46400000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81799999999999995,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.55800000000000005}]},{"osd":2,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79100000000000004,"5min":0.876,"15min":0.876},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59399999999999997,"15min":0.59599999999999997},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.80600000000000005,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84199999999999997,"5min":1.071,"15min":1.071},"last":0.436},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76200000000000001,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.496}]},{"osd":5,"last update":"Mon Mar 11 20:34:35 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.64800000000000002}]},{"osd":6,"last update":"Mon Mar 11 20:35:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.625}]},{"osd":3,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.58599999999999997}]},{"osd":4,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52600000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-11T20:35:19.140 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph pg dump --format=json 2024-03-11T20:35:19.363 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:19.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:19 smithi083 ceph-mon[24755]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:19.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:19 smithi204 ceph-mon[27258]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:20.330 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:20 smithi083 ceph-mon[24755]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:20.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:20 smithi204 ceph-mon[27258]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:21.114 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:21.115 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-03-11T20:35:21.845 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-03-11T20:35:19.568814+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48992,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912224,"statfs":{"total":767960285184,"available":767910117376,"internally_reserved":0,"allocated":6520832,"data_stored":3571156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002343"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-03-11T20:34:28.075116+0000","last_change":"2024-03-11T20:34:28.075116+0000","last_active":"2024-03-11T20:34:28.075116+0000","last_peered":"2024-03-11T20:34:28.075116+0000","last_clean":"2024-03-11T20:34:28.075116+0000","last_became_active":"2024-03-11T20:34:28.074700+0000","last_became_peered":"2024-03-11T20:34:28.074700+0000","last_unstale":"2024-03-11T20:34:28.075116+0000","last_undegraded":"2024-03-11T20:34:28.075116+0000","last_fullsized":"2024-03-11T20:34:28.075116+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-11T20:32:22.052597+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-11T20:32:22.052597+0000","last_clean_scrub_stamp":"2024-03-11T20:32:22.052597+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-13T03:32:21.620934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67900000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.502}]},{"osd":2,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.38},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.433}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.32800000000000001,"15min":0.31},"min":{"1min":0.249,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.47999999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.32800000000000001},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.54500000000000004,"5min":0.63,"15min":0.63},"last":0.377}]},{"osd":2,"last update":"Mon Mar 11 20:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.39400000000000002,"15min":0.38800000000000001},"min":{"1min":0.32500000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Mar 11 20:34:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.434,"15min":0.436},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58499999999999996,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.434,"5min":0.46300000000000002,"15min":0.46800000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66600000000000004,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.55100000000000005}]},{"osd":4,"last update":"Mon Mar 11 20:35:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.56599999999999995,"15min":0.55800000000000005},"min":{"1min":0.47499999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.442,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Mar 11 20:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.73999999999999999}]},{"osd":6,"last update":"Mon Mar 11 20:35:00 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.77000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Mar 11 20:34:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.315},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.33600000000000002,"15min":0.308},"min":{"1min":0.27500000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.433}]},{"osd":2,"last update":"Mon Mar 11 20:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.38100000000000001},"min":{"1min":0.34200000000000003,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.40300000000000002,"15min":0.39500000000000002},"min":{"1min":0.33200000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.42599999999999999}]},{"osd":3,"last update":"Mon Mar 11 20:34:54 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45700000000000002,"15min":0.45800000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.29699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Mar 11 20:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.62}]},{"osd":5,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.59999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6304,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738848,"statfs":{"total":95995035648,"available":95988580352,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36599999999999999},"min":{"1min":0.25900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36599999999999999},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.39800000000000002}]},{"osd":1,"last update":"Mon Mar 11 20:34:24 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36199999999999999,"15min":0.35999999999999999},"min":{"1min":0.28000000000000003,"5min":0.246,"15min":0.246},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.38500000000000001,"15min":0.379},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.307}]},{"osd":3,"last update":"Mon Mar 11 20:34:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40999999999999998,"15min":0.41199999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57199999999999995,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.316},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.372}]},{"osd":4,"last update":"Mon Mar 11 20:35:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.81399999999999995,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.58899999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.74099999999999999}]},{"osd":5,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.66400000000000003}]},{"osd":6,"last update":"Mon Mar 11 20:35:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.622},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.78800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75900000000000001}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.26000000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.41599999999999998,"15min":0.41499999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.41699999999999998,"15min":0.41099999999999998},"min":{"1min":0.307,"5min":0.246,"15min":0.246},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.374},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":0.308}]},{"osd":2,"last update":"Mon Mar 11 20:34:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.45600000000000002,"15min":0.45400000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.436,"15min":0.434},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Mar 11 20:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Mar 11 20:34:37 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Mar 11 20:35:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.746}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":1.143,"15min":1.143},"last":0.628},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51800000000000002}]},{"osd":1,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.35499999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73899999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58199999999999996,"15min":0.57699999999999996},"min":{"1min":0.46400000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81799999999999995,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.55800000000000005}]},{"osd":2,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79100000000000004,"5min":0.876,"15min":0.876},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59399999999999997,"15min":0.59599999999999997},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.80600000000000005,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Mar 11 20:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84199999999999997,"5min":1.071,"15min":1.071},"last":0.436},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.57399999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76200000000000001,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.496}]},{"osd":5,"last update":"Mon Mar 11 20:34:35 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.64800000000000002}]},{"osd":6,"last update":"Mon Mar 11 20:35:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.625}]},{"osd":3,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.58599999999999997}]},{"osd":4,"last update":"Mon Mar 11 20:34:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Mar 11 20:34:59 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52600000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-11T20:35:21.846 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-11T20:35:21.846 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-11T20:35:21.846 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-11T20:35:21.846 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph health --format=json 2024-03-11T20:35:22.061 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/mon.smithi083/config 2024-03-11T20:35:22.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:22 smithi204 ceph-mon[27258]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:22.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:22 smithi204 ceph-mon[27258]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:22.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:22 smithi083 ceph-mon[24755]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:22.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:22 smithi083 ceph-mon[24755]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:23.842 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:23.842 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-11T20:35:24.483 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-11T20:35:24.483 INFO:tasks.cephadm:Setup complete, yielding 2024-03-11T20:35:24.484 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-11T20:35:24.498 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:35:24.499 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph orch status' 2024-03-11T20:35:24.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:24 smithi083 ceph-mon[24755]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:24.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:24 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/778848228' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-11T20:35:24.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:24 smithi204 ceph-mon[27258]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:24.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:24 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/778848228' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-11T20:35:26.526 INFO:teuthology.orchestra.run.smithi083.stdout:Backend: cephadm 2024-03-11T20:35:26.526 INFO:teuthology.orchestra.run.smithi083.stdout:Available: Yes 2024-03-11T20:35:26.526 INFO:teuthology.orchestra.run.smithi083.stdout:Paused: No 2024-03-11T20:35:26.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:26 smithi083 ceph-mon[24755]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:26.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:26.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:26.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:26 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:26.873 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:26 smithi204 ceph-mon[27258]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:26.873 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:26.873 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:26.874 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:26 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:27.279 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph orch ps' 2024-03-11T20:35:27.884 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:27 smithi083 ceph-mon[24755]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:27.884 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:27 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:27 smithi204 ceph-mon[27258]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:27.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:27 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:28.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:28 smithi083 ceph-mon[24755]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:28.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:28 smithi204 ceph-mon[27258]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 2m ago 5m 14.5M - ba2b418f427c fc89fa4d50e8 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (5m) 65s ago 5m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 2m ago 5m 45.4M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (7m) 2m ago 7m 450M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:35:29.218 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (5m) 65s ago 5m 412M - 17.2.0 e1d6a67b021e fac7917efddd 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (7m) 2m ago 7m 43.4M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (4m) 65s ago 4m 39.8M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 2m ago 5m 16.6M - 1dbe0e931976 8913d06e5699 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (4m) 65s ago 4m 17.4M - 1dbe0e931976 52a0feb54490 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 2m ago 4m 34.1M 3216M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 2m ago 3m 37.2M 3216M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 3m 31.4M 3216M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 2m 27.3M 3216M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (2m) 65s ago 2m 33.7M 3984M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (2m) 65s ago 119s 35.3M 3984M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (95s) 65s ago 95s 30.3M 3984M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (70s) 65s ago 70s 9.95M 3984M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:35:29.219 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 2m ago 4m 54.8M - 514e6a882f6e ed14b7fa7204 2024-03-11T20:35:29.953 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph orch ls' 2024-03-11T20:35:30.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:30 smithi204 ceph-mon[27258]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:30.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:30 smithi204 ceph-mon[27258]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:30.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:30 smithi083 ceph-mon[24755]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:30.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:30 smithi083 ceph-mon[24755]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:crash 2/2 2m ago 6m * 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:mgr 2/2 2m ago 6m count:2 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:mon 2/2 2m ago 5m smithi083:172.21.15.83=smithi083;smithi204:172.21.15.204=smithi204;count:2 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:osd 8 2m ago - 2024-03-11T20:35:32.076 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-03-11T20:35:32.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:32 smithi083 ceph-mon[24755]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:32.735 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph orch host ls' 2024-03-11T20:35:32.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:32 smithi204 ceph-mon[27258]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:33.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:33 smithi083 ceph-mon[24755]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:33.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:33 smithi204 ceph-mon[27258]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:34.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:34 smithi083 ceph-mon[24755]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:34.679 INFO:teuthology.orchestra.run.smithi083.stdout:HOST ADDR LABELS STATUS 2024-03-11T20:35:34.679 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 172.21.15.83 2024-03-11T20:35:34.679 INFO:teuthology.orchestra.run.smithi083.stdout:smithi204 172.21.15.204 2024-03-11T20:35:34.679 INFO:teuthology.orchestra.run.smithi083.stdout:2 hosts in cluster 2024-03-11T20:35:34.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:34 smithi204 ceph-mon[27258]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:35.326 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph orch device ls' 2024-03-11T20:35:35.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:35 smithi204 ceph-mon[27258]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:35.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:35 smithi083 ceph-mon[24755]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:36.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:36 smithi083 ceph-mon[24755]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:36.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:36 smithi204 ceph-mon[27258]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:37.229 INFO:teuthology.orchestra.run.smithi083.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-03-11T20:35:37.229 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040148400BGN 400G LVM detected, locked 2024-03-11T20:35:37.230 INFO:teuthology.orchestra.run.smithi083.stdout:smithi204 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300GD400BGN 400G LVM detected, locked 2024-03-11T20:35:37.952 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-11T20:35:37.968 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:35:37.968 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-11T20:35:37.997 INFO:teuthology.orchestra.run.smithi083.stderr:+ systemctl stop nfs-server 2024-03-11T20:35:38.005 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi204.front.sepia.ceph.com 2024-03-11T20:35:38.005 DEBUG:teuthology.orchestra.run.smithi204:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-11T20:35:38.037 INFO:teuthology.orchestra.run.smithi204.stderr:+ systemctl stop nfs-server 2024-03-11T20:35:38.044 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-11T20:35:38.056 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:35:38.056 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph fs volume create foofs' 2024-03-11T20:35:38.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:38 smithi083 ceph-mon[24755]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:38.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:38 smithi083 ceph-mon[24755]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:38.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:38 smithi204 ceph-mon[27258]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:38.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:38 smithi204 ceph-mon[27258]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:41.174 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:40 smithi204 ceph-mon[27258]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:41.174 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:40 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-11T20:35:41.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:40 smithi083 ceph-mon[24755]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:41.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:40 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[24751]: 2024-03-11T20:35:41.637+0000 7fcdbfde8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: osdmap e61: 8 total, 8 up, 8 in 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-11T20:35:41.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: pgmap v215: 33 pgs: 15 creating+peering, 1 active+clean, 17 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: osdmap e62: 8 total, 8 up, 8 in 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: osdmap e63: 8 total, 8 up, 8 in 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: fsmap foofs:0 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: Saving service mds.foofs spec with placement count:2 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:41.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:41.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:35:41.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-11T20:35:41.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:41.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:41 smithi083 ceph-mon[24755]: Deploying daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: osdmap e61: 8 total, 8 up, 8 in 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: pgmap v215: 33 pgs: 15 creating+peering, 1 active+clean, 17 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: osdmap e62: 8 total, 8 up, 8 in 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:35:42.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: osdmap e63: 8 total, 8 up, 8 in 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: fsmap foofs:0 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: Saving service mds.foofs spec with placement count:2 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:41 smithi204 ceph-mon[27258]: Deploying daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:35:42.481 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-03-11T20:35:42.490 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-03-11T20:35:42.491 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch ls -f json 2024-03-11T20:35:43.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:43 smithi204 ceph-mon[27258]: osdmap e64: 8 total, 8 up, 8 in 2024-03-11T20:35:43.925 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:43 smithi083 ceph-mon[24755]: osdmap e64: 8 total, 8 up, 8 in 2024-03-11T20:35:44.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:44 smithi204 ceph-mon[27258]: pgmap v219: 65 pgs: 15 creating+peering, 6 active+clean, 44 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:44.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:44 smithi204 ceph-mon[27258]: osdmap e65: 8 total, 8 up, 8 in 2024-03-11T20:35:44.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:44 smithi083 ceph-mon[24755]: pgmap v219: 65 pgs: 15 creating+peering, 6 active+clean, 44 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:44.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:44 smithi083 ceph-mon[24755]: osdmap e65: 8 total, 8 up, 8 in 2024-03-11T20:35:45.578 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:45.578 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-11T20:29:00.938698Z", "last_refresh": "2024-03-11T20:32:57.099010Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-11T20:28:54.561849Z", "last_refresh": "2024-03-11T20:32:57.099075Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-11T20:28:57.624896Z", "last_refresh": "2024-03-11T20:32:57.099138Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-11T20:35:41.660192Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-11T20:35:41.655535Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-11T20:28:52.140668Z", "last_refresh": "2024-03-11T20:32:57.098938Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi204:172.21.15.204=smithi204"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-11T20:29:43.309480Z", "last_refresh": "2024-03-11T20:32:57.098796Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-11T20:28:59.682526Z", "last_refresh": "2024-03-11T20:32:57.099199Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-11T20:32:57.099334Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-11T20:28:56.256944Z", "last_refresh": "2024-03-11T20:32:57.099270Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-11T20:35:46.749 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-11T20:35:46.912 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: pgmap v221: 65 pgs: 15 creating+peering, 37 active+clean, 13 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:46.912 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:46.912 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:46.912 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:35:46.913 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-11T20:35:46.913 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:46 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: pgmap v221: 65 pgs: 15 creating+peering, 37 active+clean, 13 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-11T20:35:46.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:46 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:47.750 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch ls -f json 2024-03-11T20:35:47.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: Deploying daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:35:47.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: mds.? [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] up:boot 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: daemon mds.foofs.smithi083.motgne assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: Cluster is now healthy 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: fsmap foofs:0 1 up:standby 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:creating} 2024-03-11T20:35:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:47 smithi204 ceph-mon[27258]: daemon mds.foofs.smithi083.motgne is now active in filesystem foofs as rank 0 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: Deploying daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: mds.? [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] up:boot 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: daemon mds.foofs.smithi083.motgne assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: Cluster is now healthy 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: fsmap foofs:0 1 up:standby 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:creating} 2024-03-11T20:35:47.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:47 smithi083 ceph-mon[24755]: daemon mds.foofs.smithi083.motgne is now active in filesystem foofs as rank 0 2024-03-11T20:35:48.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:48 smithi204 ceph-mon[27258]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 172 B/s wr, 0 op/s 2024-03-11T20:35:48.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:48 smithi204 ceph-mon[27258]: mds.? [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] up:active 2024-03-11T20:35:48.924 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:48 smithi204 ceph-mon[27258]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 2024-03-11T20:35:48.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:48 smithi083 ceph-mon[24755]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 172 B/s wr, 0 op/s 2024-03-11T20:35:48.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:48 smithi083 ceph-mon[24755]: mds.? [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] up:active 2024-03-11T20:35:48.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:48 smithi083 ceph-mon[24755]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 2024-03-11T20:35:49.883 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:49.883 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-11T20:29:00.938698Z", "last_refresh": "2024-03-11T20:32:57.099010Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-11T20:28:54.561849Z", "last_refresh": "2024-03-11T20:32:57.099075Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-11T20:28:57.624896Z", "last_refresh": "2024-03-11T20:32:57.099138Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-11T20:35:41.660192Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-11T20:35:41.655535Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-11T20:28:52.140668Z", "last_refresh": "2024-03-11T20:32:57.098938Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi204:172.21.15.204=smithi204"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-11T20:29:43.309480Z", "last_refresh": "2024-03-11T20:32:57.098796Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-11T20:28:59.682526Z", "last_refresh": "2024-03-11T20:32:57.099199Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-11T20:32:57.099334Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-11T20:28:56.256944Z", "last_refresh": "2024-03-11T20:32:57.099270Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:boot 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 129 B/s wr, 0 op/s 2024-03-11T20:35:49.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:49 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:boot 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:35:50.110 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:35:50.111 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:35:50.111 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 129 B/s wr, 0 op/s 2024-03-11T20:35:50.111 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:49 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:51.036 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-11T20:35:51.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:51 smithi083 ceph-mon[24755]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:51.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:51 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:51.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:51 smithi204 ceph-mon[27258]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:51.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:51 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:52.037 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch ls -f json 2024-03-11T20:35:52.326 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:52 smithi204 ceph-mon[27258]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-03-11T20:35:52.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:52 smithi083 ceph-mon[24755]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-03-11T20:35:54.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:54 smithi083 ceph-mon[24755]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-11T20:35:54.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:54.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:54 smithi204 ceph-mon[27258]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-11T20:35:54.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:55.612 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:35:55.612 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-11T20:29:00.938698Z", "last_refresh": "2024-03-11T20:35:55.591081Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-11T20:28:54.561849Z", "last_refresh": "2024-03-11T20:35:50.801308Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-11T20:28:57.624896Z", "last_refresh": "2024-03-11T20:35:55.591208Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-11T20:35:41.660192Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-11T20:35:41.655535Z", "last_refresh": "2024-03-11T20:35:50.801901Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-11T20:28:52.140668Z", "last_refresh": "2024-03-11T20:35:50.801454Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi204:172.21.15.204=smithi204"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-11T20:29:43.309480Z", "last_refresh": "2024-03-11T20:35:50.801542Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-11T20:28:59.682526Z", "last_refresh": "2024-03-11T20:35:50.801604Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-11T20:35:50.801664Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-11T20:28:56.256944Z", "last_refresh": "2024-03-11T20:35:55.591344Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-11T20:35:56.393 INFO:tasks.cephadm:mds.foofs has 2/2 2024-03-11T20:35:56.393 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-11T20:35:56.402 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:35:56.403 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-03-11T20:35:56.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-11T20:35:56.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:56.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:56.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:56.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:56 smithi083 ceph-mon[24755]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:35:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:56 smithi204 ceph-mon[27258]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:35:58.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:58 smithi204 ceph-mon[27258]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-11T20:35:58.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:58 smithi083 ceph-mon[24755]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-11T20:35:59.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:59 smithi204 ceph-mon[27258]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:59.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:35:59 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-11T20:35:59.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:59 smithi083 ceph-mon[24755]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:35:59.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:35:59 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-11T20:36:00.119 INFO:teuthology.orchestra.run.smithi083.stdout:NFS Cluster Created Successfully 2024-03-11T20:36:00.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-11T20:36:00.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:00 smithi083 ceph-mon[24755]: osdmap e66: 8 total, 8 up, 8 in 2024-03-11T20:36:00.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:00 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-11T20:36:00.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:00 smithi083 ceph-mon[24755]: pgmap v229: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-03-11T20:36:00.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-11T20:36:00.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:00 smithi204 ceph-mon[27258]: osdmap e66: 8 total, 8 up, 8 in 2024-03-11T20:36:00.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:00 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-11T20:36:00.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:00 smithi204 ceph-mon[27258]: pgmap v229: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-03-11T20:36:00.771 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-03-11T20:36:01.343 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: osdmap e67: 8 total, 8 up, 8 in 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Saving service nfs.foo spec with placement count:2 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:36:01.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Rados config object exists: conf-nfs.foo 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq-rgw 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:01 smithi083 ceph-mon[24755]: Deploying daemon nfs.foo.0.0.smithi083.rbftzq on smithi083 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: osdmap e67: 8 total, 8 up, 8 in 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Saving service nfs.foo spec with placement count:2 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Rados config object exists: conf-nfs.foo 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq-rgw 2024-03-11T20:36:01.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:36:01.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-11T20:36:01.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:01.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:01 smithi204 ceph-mon[27258]: Deploying daemon nfs.foo.0.0.smithi083.rbftzq on smithi083 2024-03-11T20:36:02.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:02 smithi083 ceph-mon[24755]: osdmap e68: 8 total, 8 up, 8 in 2024-03-11T20:36:02.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:02 smithi083 ceph-mon[24755]: pgmap v232: 97 pgs: 9 creating+peering, 9 unknown, 79 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:36:02.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:02 smithi204 ceph-mon[27258]: osdmap e68: 8 total, 8 up, 8 in 2024-03-11T20:36:02.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:02 smithi204 ceph-mon[27258]: pgmap v232: 97 pgs: 9 creating+peering, 9 unknown, 79 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:36:03.338 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:03 smithi083 ceph-mon[24755]: mgrmap e20: smithi083.jyxogi(active, since 5m), standbys: smithi204.qknpir 2024-03-11T20:36:03.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:03 smithi204 ceph-mon[27258]: mgrmap e20: smithi083.jyxogi(active, since 5m), standbys: smithi204.qknpir 2024-03-11T20:36:04.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:04 smithi083 ceph-mon[24755]: pgmap v233: 97 pgs: 9 creating+peering, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:36:04.288 INFO:teuthology.orchestra.run.smithi083.stderr:Invalid command: Unexpected argument '--clusterid' 2024-03-11T20:36:04.288 INFO:teuthology.orchestra.run.smithi083.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-03-11T20:36:04.288 INFO:teuthology.orchestra.run.smithi083.stderr:Error EINVAL: invalid command 2024-03-11T20:36:04.424 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:04 smithi204 ceph-mon[27258]: pgmap v233: 97 pgs: 9 creating+peering, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout: "bind": "/fake", 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout: "fs": "foofs", 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout: "path": "/", 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster": "foo", 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout: "mode": "RW" 2024-03-11T20:36:04.640 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:36:05.355 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: Creating key for client.nfs.foo.1.0.smithi204.brmoam 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:36:05.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:05.662 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:05.662 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-11T20:36:05.662 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:05 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: Creating key for client.nfs.foo.1.0.smithi204.brmoam 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-11T20:36:05.675 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:05 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-11T20:36:06.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:06 smithi204 ceph-mon[27258]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-11T20:36:06.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:06 smithi083 ceph-mon[24755]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: Rados config object exists: conf-nfs.foo 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: Creating key for client.nfs.foo.1.0.smithi204.brmoam-rgw 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: Deploying daemon nfs.foo.1.0.smithi204.brmoam on smithi204 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:08 smithi204 ceph-mon[27258]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: Rados config object exists: conf-nfs.foo 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: Creating key for client.nfs.foo.1.0.smithi204.brmoam-rgw 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:08.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: Deploying daemon nfs.foo.1.0.smithi204.brmoam on smithi204 2024-03-11T20:36:08.443 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:08.443 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:08 smithi083 ceph-mon[24755]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-11T20:36:09.596 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:09 smithi204 ceph-mon[27258]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:09.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:09 smithi083 ceph-mon[24755]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]: dispatch 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-11T20:36:10.613 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:36:10.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:36:10.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:10.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:10.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:10.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:10 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:10.629 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]: dispatch 2024-03-11T20:36:10.629 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-03-11T20:36:10.629 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-03-11T20:36:10.629 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:10.630 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:10 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]': finished 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-03-11T20:36:11.588 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:11 smithi204 ceph-mon[27258]: osdmap e69: 8 total, 8 up, 8 in 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [7, 2]}]': finished 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-03-11T20:36:11.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:11 smithi083 ceph-mon[24755]: osdmap e69: 8 total, 8 up, 8 in 2024-03-11T20:36:12.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:12 smithi083 ceph-mon[24755]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:12.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:12 smithi083 ceph-mon[24755]: osdmap e70: 8 total, 8 up, 8 in 2024-03-11T20:36:12.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:12 smithi083 ceph-mon[24755]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2.4 KiB/s wr, 5 op/s 2024-03-11T20:36:12.526 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:12 smithi204 ceph-mon[27258]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:12.526 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:12 smithi204 ceph-mon[27258]: osdmap e70: 8 total, 8 up, 8 in 2024-03-11T20:36:12.526 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:12 smithi204 ceph-mon[27258]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2.4 KiB/s wr, 5 op/s 2024-03-11T20:36:13.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:13 smithi083 ceph-mon[24755]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:13.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:13.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:13 smithi204 ceph-mon[27258]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:13.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:13 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:14.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:14 smithi083 ceph-mon[24755]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-11T20:36:14.674 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:14 smithi204 ceph-mon[27258]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-11T20:36:15.184 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-03-11T20:36:15.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:15.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:15.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:15.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:15.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:15 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:15.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:15 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:15.987 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-11T20:36:15.996 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:36:15.997 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-03-11T20:36:16.024 INFO:teuthology.orchestra.run.smithi083.stderr:+ mkdir /mnt/foo 2024-03-11T20:36:16.031 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-03-11T20:36:16.097 INFO:teuthology.orchestra.run.smithi083.stderr:++ hostname 2024-03-11T20:36:16.097 INFO:teuthology.orchestra.run.smithi083.stderr:+ mount -t nfs smithi083:/fake /mnt/foo -o sync 2024-03-11T20:36:16.533 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-03-11T20:36:16.599 INFO:teuthology.orchestra.run.smithi083.stderr:+ echo test 2024-03-11T20:36:16.610 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-03-11T20:36:16.673 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:16 smithi083 ceph-mon[24755]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:16.673 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:16 smithi083 ceph-mon[24755]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-11T20:36:16.675 INFO:teuthology.orchestra.run.smithi083.stderr:+ sync 2024-03-11T20:36:16.828 INFO:teuthology.run_tasks:Running task parallel... 2024-03-11T20:36:16.838 INFO:teuthology.task.parallel:starting parallel... 2024-03-11T20:36:16.839 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-11T20:36:16.839 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-11T20:36:16.840 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:36:16.840 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-11T20:36:16.840 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-11T20:36:16.840 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-03-11T20:36:16.841 INFO:teuthology.task.exec:Executing custom commands... 2024-03-11T20:36:16.841 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:36:16.841 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-03-11T20:36:16.899 INFO:teuthology.orchestra.run.smithi083.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-11T20:36:16.899 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:36:16.899 INFO:teuthology.orchestra.run.smithi083.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-11T20:36:16.900 INFO:teuthology.orchestra.run.smithi083.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-11T20:36:16.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:16 smithi204 ceph-mon[27258]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:16.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:16 smithi204 ceph-mon[27258]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-11T20:36:17.001 INFO:teuthology.orchestra.run.smithi083.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-11T20:36:17.001 INFO:teuthology.orchestra.run.smithi083.stdout:releasing clients 2024-03-11T20:36:18.001 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222 42.99 MB/sec warmup 1 sec latency 31.495 ms 2024-03-11T20:36:18.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:18 smithi083 ceph-mon[24755]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-11T20:36:18.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:18 smithi204 ceph-mon[27258]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-03-11T20:36:19.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 441 38.64 MB/sec warmup 2 sec latency 23.313 ms 2024-03-11T20:36:20.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 559 31.94 MB/sec warmup 3 sec latency 32.257 ms 2024-03-11T20:36:20.595 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-11T20:36:20.874 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:20 smithi083 ceph-mon[24755]: pgmap v243: 97 pgs: 97 active+clean; 17 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.8 MiB/s wr, 40 op/s 2024-03-11T20:36:20.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:20 smithi204 ceph-mon[27258]: pgmap v243: 97 pgs: 97 active+clean; 17 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.8 MiB/s wr, 40 op/s 2024-03-11T20:36:21.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 771 26.71 MB/sec warmup 4 sec latency 31.242 ms 2024-03-11T20:36:22.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1086 22.92 MB/sec warmup 5 sec latency 46.293 ms 2024-03-11T20:36:22.653 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:22 smithi083 ceph-mon[24755]: pgmap v244: 97 pgs: 97 active+clean; 86 MiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 8.6 MiB/s wr, 348 op/s 2024-03-11T20:36:22.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:22 smithi204 ceph-mon[27258]: pgmap v244: 97 pgs: 97 active+clean; 86 MiB data, 145 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 8.6 MiB/s wr, 348 op/s 2024-03-11T20:36:23.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1917 22.00 MB/sec warmup 6 sec latency 55.597 ms 2024-03-11T20:36:24.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 2885 23.07 MB/sec warmup 7 sec latency 36.995 ms 2024-03-11T20:36:24.668 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:24 smithi083 ceph-mon[24755]: pgmap v245: 97 pgs: 97 active+clean; 98 MiB data, 150 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 8.4 MiB/s wr, 321 op/s 2024-03-11T20:36:24.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:24 smithi204 ceph-mon[27258]: pgmap v245: 97 pgs: 97 active+clean; 98 MiB data, 150 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 8.4 MiB/s wr, 321 op/s 2024-03-11T20:36:25.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 3508 23.62 MB/sec warmup 8 sec latency 18.803 ms 2024-03-11T20:36:25.591 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-11T20:36:26.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 4018 22.25 MB/sec warmup 9 sec latency 18.903 ms 2024-03-11T20:36:26.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:26 smithi083 ceph-mon[24755]: pgmap v246: 97 pgs: 97 active+clean; 120 MiB data, 206 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 11 MiB/s wr, 577 op/s 2024-03-11T20:36:26.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:26 smithi204 ceph-mon[27258]: pgmap v246: 97 pgs: 97 active+clean; 120 MiB data, 206 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 11 MiB/s wr, 577 op/s 2024-03-11T20:36:27.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 4824 21.38 MB/sec warmup 10 sec latency 21.718 ms 2024-03-11T20:36:28.004 INFO:teuthology.orchestra.run.smithi083.stdout: 5 5709 20.87 MB/sec warmup 11 sec latency 21.633 ms 2024-03-11T20:36:28.224 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:28 smithi083 ceph-mon[24755]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 13 MiB/s wr, 706 op/s 2024-03-11T20:36:28.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:28 smithi204 ceph-mon[27258]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 13 MiB/s wr, 706 op/s 2024-03-11T20:36:29.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 6378 21.48 MB/sec warmup 12 sec latency 51.442 ms 2024-03-11T20:36:30.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7327 22.55 MB/sec warmup 13 sec latency 11.664 ms 2024-03-11T20:36:30.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:30 smithi083 ceph-mon[24755]: pgmap v248: 97 pgs: 97 active+clean; 132 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 14 MiB/s wr, 753 op/s 2024-03-11T20:36:30.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:30 smithi204 ceph-mon[27258]: pgmap v248: 97 pgs: 97 active+clean; 132 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 14 MiB/s wr, 753 op/s 2024-03-11T20:36:31.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 8089 21.58 MB/sec warmup 14 sec latency 20.636 ms 2024-03-11T20:36:31.163 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph mgr module enable nfs --force' 2024-03-11T20:36:32.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 9011 21.56 MB/sec warmup 15 sec latency 14.757 ms 2024-03-11T20:36:32.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:32 smithi083 ceph-mon[24755]: pgmap v249: 97 pgs: 97 active+clean; 140 MiB data, 313 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 15 MiB/s wr, 1.03k op/s 2024-03-11T20:36:32.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:32 smithi204 ceph-mon[27258]: pgmap v249: 97 pgs: 97 active+clean; 140 MiB data, 313 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 15 MiB/s wr, 1.03k op/s 2024-03-11T20:36:33.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 10053 22.26 MB/sec warmup 16 sec latency 20.128 ms 2024-03-11T20:36:34.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 10915 22.97 MB/sec warmup 17 sec latency 67.788 ms 2024-03-11T20:36:34.880 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:34 smithi083 ceph-mon[24755]: pgmap v250: 97 pgs: 97 active+clean; 140 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.8 MiB/s wr, 782 op/s 2024-03-11T20:36:34.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:34 smithi204 ceph-mon[27258]: pgmap v250: 97 pgs: 97 active+clean; 140 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.8 MiB/s wr, 782 op/s 2024-03-11T20:36:35.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11575 22.05 MB/sec warmup 18 sec latency 27.643 ms 2024-03-11T20:36:35.662 INFO:teuthology.orchestra.run.smithi083.stderr:module 'nfs' is already enabled 2024-03-11T20:36:35.885 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:35 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4229473207' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-11T20:36:35.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:35 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4229473207' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-11T20:36:36.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 12587 22.05 MB/sec warmup 19 sec latency 22.976 ms 2024-03-11T20:36:36.871 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-11T20:36:36.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:36 smithi083 ceph-mon[24755]: pgmap v251: 97 pgs: 97 active+clean; 153 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-03-11T20:36:36.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:36 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4229473207' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-11T20:36:36.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:36 smithi083 ceph-mon[24755]: mgrmap e21: smithi083.jyxogi(active, since 6m), standbys: smithi204.qknpir 2024-03-11T20:36:36.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:36 smithi204 ceph-mon[27258]: pgmap v251: 97 pgs: 97 active+clean; 153 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-03-11T20:36:36.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:36 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4229473207' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-11T20:36:36.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:36 smithi204 ceph-mon[27258]: mgrmap e21: smithi083.jyxogi(active, since 6m), standbys: smithi204.qknpir 2024-03-11T20:36:37.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13320 21.92 MB/sec warmup 20 sec latency 43.949 ms 2024-03-11T20:36:38.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14181 22.55 MB/sec warmup 21 sec latency 22.939 ms 2024-03-11T20:36:38.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:38 smithi083 ceph-mon[24755]: pgmap v252: 97 pgs: 97 active+clean; 157 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-03-11T20:36:38.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:38 smithi204 ceph-mon[27258]: pgmap v252: 97 pgs: 97 active+clean; 157 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-03-11T20:36:39.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14657 22.13 MB/sec warmup 22 sec latency 25.562 ms 2024-03-11T20:36:40.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15181 21.45 MB/sec warmup 23 sec latency 45.890 ms 2024-03-11T20:36:40.648 INFO:teuthology.orchestra.run.smithi083.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:36:40.679 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:40 smithi083 ceph-mon[24755]: pgmap v253: 97 pgs: 97 active+clean; 160 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 994 op/s 2024-03-11T20:36:40.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:40 smithi204 ceph-mon[27258]: pgmap v253: 97 pgs: 97 active+clean; 160 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 994 op/s 2024-03-11T20:36:41.005 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15994 21.41 MB/sec warmup 24 sec latency 30.716 ms 2024-03-11T20:36:41.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:41 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:41.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:36:41.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:36:41.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:36:41.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:36:41.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:41 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:36:41.996 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-11T20:36:41.997 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:36:41.997 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-03-11T20:36:42.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 16932 21.48 MB/sec warmup 25 sec latency 21.455 ms 2024-03-11T20:36:42.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:42 smithi083 ceph-mon[24755]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:36:42.692 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:42 smithi083 ceph-mon[24755]: pgmap v254: 97 pgs: 97 active+clean; 167 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-03-11T20:36:42.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:42 smithi204 ceph-mon[27258]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b 2024-03-11T20:36:42.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:42 smithi204 ceph-mon[27258]: pgmap v254: 97 pgs: 97 active+clean; 167 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.21k op/s 2024-03-11T20:36:43.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 17707 21.88 MB/sec warmup 26 sec latency 19.095 ms 2024-03-11T20:36:44.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18170 21.57 MB/sec warmup 27 sec latency 33.044 ms 2024-03-11T20:36:44.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:44 smithi083 ceph-mon[24755]: pgmap v255: 97 pgs: 97 active+clean; 168 MiB data, 540 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 931 op/s 2024-03-11T20:36:44.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:44 smithi204 ceph-mon[27258]: pgmap v255: 97 pgs: 97 active+clean; 168 MiB data, 540 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 931 op/s 2024-03-11T20:36:45.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18711 21.01 MB/sec warmup 28 sec latency 23.978 ms 2024-03-11T20:36:46.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 19270 20.88 MB/sec warmup 29 sec latency 25.220 ms 2024-03-11T20:36:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:36:46.713 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:46 smithi083 ceph-mon[24755]: pgmap v256: 97 pgs: 97 active+clean; 172 MiB data, 590 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-03-11T20:36:46.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:46 smithi204 ceph-mon[27258]: pgmap v256: 97 pgs: 97 active+clean; 172 MiB data, 590 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-03-11T20:36:47.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20024 20.47 MB/sec warmup 30 sec latency 37.533 ms 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 32s ago 7m 22.6M - ba2b418f427c fc89fa4d50e8 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 32s ago 7m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (6m) 34s ago 6m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 32s ago 6m 51.9M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (61s) 32s ago 60s 13.6M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (58s) 34s ago 58s 14.4M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:36:47.138 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (8m) 32s ago 8m 462M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (6m) 34s ago 6m 412M - 17.2.0 e1d6a67b021e fac7917efddd 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 32s ago 8m 47.6M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (6m) 34s ago 6m 37.5M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (43s) 32s ago 42s 58.2M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (37s) 34s ago 37s 46.7M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 32s ago 6m 18.8M - 1dbe0e931976 8913d06e5699 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (6m) 34s ago 6m 17.4M - 1dbe0e931976 52a0feb54490 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 32s ago 5m 43.3M 3216M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 32s ago 5m 45.4M 3216M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 32s ago 4m 39.1M 3216M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 32s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (3m) 34s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (3m) 34s ago 3m 42.2M 3984M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (2m) 34s ago 2m 38.5M 3984M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (2m) 34s ago 2m 39.8M 3984M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:36:47.139 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 32s ago 5m 63.5M - 514e6a882f6e ed14b7fa7204 2024-03-11T20:36:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:47 smithi083 ceph-mon[24755]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:47.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:47 smithi083 ceph-mon[24755]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:47.759 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:36:47.759 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-11T20:36:47.760 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:36:47.761 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:36:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:47 smithi204 ceph-mon[27258]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:47.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:47 smithi204 ceph-mon[27258]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:48.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20488 20.33 MB/sec warmup 31 sec latency 22.535 ms 2024-03-11T20:36:48.274 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:36:48.274 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b", 2024-03-11T20:36:48.275 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:36:48.275 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:36:48.275 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-03-11T20:36:48.275 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b image" 2024-03-11T20:36:48.275 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:36:48.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:48 smithi083 ceph-mon[24755]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:48.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:48 smithi083 ceph-mon[24755]: pgmap v257: 97 pgs: 97 active+clean; 178 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-03-11T20:36:48.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:48 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3712603553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:36:48.872 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:36:48.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:48 smithi204 ceph-mon[27258]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:48.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:48 smithi204 ceph-mon[27258]: pgmap v257: 97 pgs: 97 active+clean; 178 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-03-11T20:36:48.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:48 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3712603553' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:36:49.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21194 20.64 MB/sec warmup 32 sec latency 23.747 ms 2024-03-11T20:36:49.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:49 smithi083 ceph-mon[24755]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:49.766 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:49 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3362017195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:36:49.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:49 smithi204 ceph-mon[27258]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:36:49.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:49 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3362017195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:36:50.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21688 20.54 MB/sec warmup 33 sec latency 22.994 ms 2024-03-11T20:36:50.774 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:50 smithi083 ceph-mon[24755]: pgmap v258: 97 pgs: 97 active+clean; 176 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 821 op/s 2024-03-11T20:36:50.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:50 smithi204 ceph-mon[27258]: pgmap v258: 97 pgs: 97 active+clean; 176 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 821 op/s 2024-03-11T20:36:51.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22105 20.02 MB/sec warmup 34 sec latency 28.312 ms 2024-03-11T20:36:52.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22523 19.76 MB/sec warmup 35 sec latency 33.172 ms 2024-03-11T20:36:52.796 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:52 smithi083 ceph-mon[24755]: pgmap v259: 97 pgs: 97 active+clean; 187 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-03-11T20:36:52.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:52 smithi204 ceph-mon[27258]: pgmap v259: 97 pgs: 97 active+clean; 187 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-03-11T20:36:53.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23190 19.61 MB/sec warmup 36 sec latency 44.240 ms 2024-03-11T20:36:54.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23746 19.31 MB/sec warmup 37 sec latency 30.137 ms 2024-03-11T20:36:54.810 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:54 smithi083 ceph-mon[24755]: pgmap v260: 97 pgs: 97 active+clean; 189 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.3 MiB/s wr, 745 op/s 2024-03-11T20:36:54.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:54 smithi204 ceph-mon[27258]: pgmap v260: 97 pgs: 97 active+clean; 189 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.3 MiB/s wr, 745 op/s 2024-03-11T20:36:55.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 24493 19.64 MB/sec warmup 38 sec latency 22.249 ms 2024-03-11T20:36:56.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25103 19.77 MB/sec warmup 39 sec latency 33.504 ms 2024-03-11T20:36:56.834 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:56 smithi083 ceph-mon[24755]: pgmap v261: 97 pgs: 97 active+clean; 191 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-03-11T20:36:56.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:56 smithi204 ceph-mon[27258]: pgmap v261: 97 pgs: 97 active+clean; 191 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-03-11T20:36:57.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25557 19.38 MB/sec warmup 40 sec latency 32.947 ms 2024-03-11T20:36:58.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 26348 19.44 MB/sec warmup 41 sec latency 14.318 ms 2024-03-11T20:36:58.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:36:58 smithi083 ceph-mon[24755]: pgmap v262: 97 pgs: 97 active+clean; 194 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 850 op/s 2024-03-11T20:36:58.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:36:58 smithi204 ceph-mon[27258]: pgmap v262: 97 pgs: 97 active+clean; 194 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 850 op/s 2024-03-11T20:36:59.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 27178 19.21 MB/sec warmup 42 sec latency 25.101 ms 2024-03-11T20:37:00.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 27935 19.52 MB/sec warmup 43 sec latency 25.308 ms 2024-03-11T20:37:00.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:00 smithi083 ceph-mon[24755]: pgmap v263: 97 pgs: 97 active+clean; 199 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.9 MiB/s wr, 740 op/s 2024-03-11T20:37:00.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:00 smithi204 ceph-mon[27258]: pgmap v263: 97 pgs: 97 active+clean; 199 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.9 MiB/s wr, 740 op/s 2024-03-11T20:37:01.006 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28788 19.75 MB/sec warmup 44 sec latency 26.399 ms 2024-03-11T20:37:02.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 29542 19.54 MB/sec warmup 45 sec latency 18.126 ms 2024-03-11T20:37:02.864 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:02 smithi083 ceph-mon[24755]: pgmap v264: 97 pgs: 97 active+clean; 202 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 952 op/s 2024-03-11T20:37:02.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:02 smithi204 ceph-mon[27258]: pgmap v264: 97 pgs: 97 active+clean; 202 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 952 op/s 2024-03-11T20:37:03.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 30492 19.54 MB/sec warmup 46 sec latency 34.915 ms 2024-03-11T20:37:04.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 31036 19.48 MB/sec warmup 47 sec latency 35.050 ms 2024-03-11T20:37:04.666 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:04 smithi083 ceph-mon[24755]: pgmap v265: 97 pgs: 97 active+clean; 207 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-03-11T20:37:04.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:04 smithi204 ceph-mon[27258]: pgmap v265: 97 pgs: 97 active+clean; 207 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-03-11T20:37:05.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 31513 19.48 MB/sec warmup 48 sec latency 31.707 ms 2024-03-11T20:37:06.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32088 19.60 MB/sec warmup 49 sec latency 38.536 ms 2024-03-11T20:37:06.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:06 smithi083 ceph-mon[24755]: pgmap v266: 97 pgs: 97 active+clean; 212 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 888 op/s 2024-03-11T20:37:06.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:06 smithi204 ceph-mon[27258]: pgmap v266: 97 pgs: 97 active+clean; 212 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 888 op/s 2024-03-11T20:37:07.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32400 19.31 MB/sec warmup 50 sec latency 24.238 ms 2024-03-11T20:37:08.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32934 19.05 MB/sec warmup 51 sec latency 31.246 ms 2024-03-11T20:37:08.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:08 smithi083 ceph-mon[24755]: pgmap v267: 97 pgs: 97 active+clean; 216 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 942 op/s 2024-03-11T20:37:08.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:08 smithi204 ceph-mon[27258]: pgmap v267: 97 pgs: 97 active+clean; 216 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 942 op/s 2024-03-11T20:37:09.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 33743 19.08 MB/sec warmup 52 sec latency 25.815 ms 2024-03-11T20:37:09.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:37:09.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:37:09.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:37:09.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:37:10.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 34457 18.97 MB/sec warmup 53 sec latency 23.217 ms 2024-03-11T20:37:10.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:10 smithi083 ceph-mon[24755]: pgmap v268: 97 pgs: 97 active+clean; 219 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 846 op/s 2024-03-11T20:37:10.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:10 smithi204 ceph-mon[27258]: pgmap v268: 97 pgs: 97 active+clean; 219 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 846 op/s 2024-03-11T20:37:11.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 35073 19.11 MB/sec warmup 54 sec latency 24.912 ms 2024-03-11T20:37:12.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 35739 19.24 MB/sec warmup 55 sec latency 25.284 ms 2024-03-11T20:37:12.832 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:12 smithi083 ceph-mon[24755]: pgmap v269: 97 pgs: 97 active+clean; 225 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 972 op/s 2024-03-11T20:37:12.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:12 smithi204 ceph-mon[27258]: pgmap v269: 97 pgs: 97 active+clean; 225 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 972 op/s 2024-03-11T20:37:13.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 36244 18.98 MB/sec warmup 56 sec latency 41.040 ms 2024-03-11T20:37:14.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 36666 18.83 MB/sec warmup 57 sec latency 48.395 ms 2024-03-11T20:37:14.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:14 smithi083 ceph-mon[24755]: pgmap v270: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 753 op/s 2024-03-11T20:37:14.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:14 smithi204 ceph-mon[27258]: pgmap v270: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 753 op/s 2024-03-11T20:37:15.007 INFO:teuthology.orchestra.run.smithi083.stdout: 5 37291 18.75 MB/sec warmup 58 sec latency 34.130 ms 2024-03-11T20:37:16.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 38004 18.66 MB/sec warmup 59 sec latency 130.122 ms 2024-03-11T20:37:16.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:16 smithi083 ceph-mon[24755]: pgmap v271: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 806 op/s 2024-03-11T20:37:16.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:16 smithi204 ceph-mon[27258]: pgmap v271: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 806 op/s 2024-03-11T20:37:17.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 38543 18.74 MB/sec warmup 60 sec latency 138.216 ms 2024-03-11T20:37:18.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39238 18.89 MB/sec warmup 61 sec latency 97.343 ms 2024-03-11T20:37:18.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:18 smithi083 ceph-mon[24755]: pgmap v272: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 911 op/s 2024-03-11T20:37:18.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:18 smithi204 ceph-mon[27258]: pgmap v272: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 911 op/s 2024-03-11T20:37:19.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39648 18.67 MB/sec warmup 62 sec latency 51.833 ms 2024-03-11T20:37:19.423 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:37:20.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40096 18.48 MB/sec warmup 63 sec latency 43.102 ms 2024-03-11T20:37:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:37:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 65s ago 7m 22.6M - ba2b418f427c fc89fa4d50e8 2024-03-11T20:37:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 65s ago 7m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:37:20.496 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (6m) 67s ago 6m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 65s ago 7m 51.9M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (94s) 65s ago 94s 13.6M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (92s) 67s ago 91s 14.4M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (8m) 65s ago 8m 462M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (6m) 67s ago 6m 412M - 17.2.0 e1d6a67b021e fac7917efddd 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 65s ago 9m 47.6M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (6m) 67s ago 6m 37.5M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (77s) 65s ago 76s 58.2M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (71s) 67s ago 70s 46.7M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 65s ago 7m 18.8M - 1dbe0e931976 8913d06e5699 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (6m) 67s ago 6m 17.4M - 1dbe0e931976 52a0feb54490 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 65s ago 6m 43.3M 3216M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 65s ago 5m 45.4M 3216M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 65s ago 5m 39.1M 3216M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:37:20.497 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 65s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:37:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (4m) 67s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:37:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (3m) 67s ago 3m 42.2M 3984M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:37:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (3m) 67s ago 3m 38.5M 3984M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:37:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (3m) 67s ago 3m 39.8M 3984M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:37:20.498 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 65s ago 6m 63.5M - 514e6a882f6e ed14b7fa7204 2024-03-11T20:37:20.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:20 smithi083 ceph-mon[24755]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:20.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:20 smithi083 ceph-mon[24755]: pgmap v273: 97 pgs: 97 active+clean; 236 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-03-11T20:37:20.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:20 smithi204 ceph-mon[27258]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:20.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:20 smithi204 ceph-mon[27258]: pgmap v273: 97 pgs: 97 active+clean; 236 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-03-11T20:37:21.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40618 18.43 MB/sec warmup 64 sec latency 30.080 ms 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:37:21.118 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:37:21.119 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:37:21.613 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:37:21.613 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b", 2024-03-11T20:37:21.614 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:37:21.614 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:37:21.614 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-03-11T20:37:21.614 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:2b0101eb74d60610d544cb8a473e23a35ac1cc0b image" 2024-03-11T20:37:21.614 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:37:21.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:21 smithi083 ceph-mon[24755]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:21.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:21 smithi083 ceph-mon[24755]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:21.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:21 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2265224098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:37:21.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:21 smithi204 ceph-mon[27258]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:21.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:21 smithi204 ceph-mon[27258]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:21.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:21 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2265224098' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:37:22.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 41345 18.30 MB/sec warmup 65 sec latency 25.039 ms 2024-03-11T20:37:22.255 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: pgmap v274: 97 pgs: 97 active+clean; 253 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.9 MiB/s wr, 974 op/s 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: Upgrade: Target is version 19.0.0-1979-g2b0101eb (unknown) 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379'] 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi083.jyxogi) 2024-03-11T20:37:22.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:22 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/4155339531' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:37:23.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42046 18.42 MB/sec warmup 66 sec latency 30.372 ms 2024-03-11T20:37:23.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: pgmap v274: 97 pgs: 97 active+clean; 253 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.9 MiB/s wr, 974 op/s 2024-03-11T20:37:23.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: Upgrade: Target is version 19.0.0-1979-g2b0101eb (unknown) 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379'] 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi083.jyxogi) 2024-03-11T20:37:23.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:22 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/4155339531' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:37:23.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:23 smithi083 ceph-mon[24755]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 on smithi204 2024-03-11T20:37:24.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42750 18.62 MB/sec warmup 67 sec latency 35.008 ms 2024-03-11T20:37:24.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:23 smithi204 ceph-mon[27258]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 on smithi204 2024-03-11T20:37:24.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:24 smithi083 ceph-mon[24755]: pgmap v275: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 744 op/s 2024-03-11T20:37:25.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43166 18.42 MB/sec warmup 68 sec latency 31.989 ms 2024-03-11T20:37:25.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:24 smithi204 ceph-mon[27258]: pgmap v275: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 744 op/s 2024-03-11T20:37:26.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43549 18.23 MB/sec warmup 69 sec latency 60.025 ms 2024-03-11T20:37:26.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:26 smithi083 ceph-mon[24755]: pgmap v276: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 798 op/s 2024-03-11T20:37:27.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43954 18.17 MB/sec warmup 70 sec latency 78.352 ms 2024-03-11T20:37:27.175 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:26 smithi204 ceph-mon[27258]: pgmap v276: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 798 op/s 2024-03-11T20:37:28.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 44595 18.04 MB/sec warmup 71 sec latency 38.293 ms 2024-03-11T20:37:28.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:28 smithi083 ceph-mon[24755]: pgmap v277: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 883 op/s 2024-03-11T20:37:28.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:28 smithi204 ceph-mon[27258]: pgmap v277: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 883 op/s 2024-03-11T20:37:29.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 45383 18.12 MB/sec warmup 72 sec latency 22.130 ms 2024-03-11T20:37:30.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46389 18.42 MB/sec warmup 73 sec latency 21.341 ms 2024-03-11T20:37:30.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:30 smithi083 ceph-mon[24755]: pgmap v278: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.9 MiB/s wr, 723 op/s 2024-03-11T20:37:30.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:30 smithi204 ceph-mon[27258]: pgmap v278: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.9 MiB/s wr, 723 op/s 2024-03-11T20:37:31.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 47112 18.30 MB/sec warmup 74 sec latency 16.734 ms 2024-03-11T20:37:32.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 48024 18.34 MB/sec warmup 75 sec latency 26.065 ms 2024-03-11T20:37:32.712 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:32 smithi083 ceph-mon[24755]: pgmap v279: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-11T20:37:32.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:32 smithi204 ceph-mon[27258]: pgmap v279: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-11T20:37:33.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 48947 18.44 MB/sec warmup 76 sec latency 20.151 ms 2024-03-11T20:37:34.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 49889 18.70 MB/sec warmup 77 sec latency 22.575 ms 2024-03-11T20:37:34.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:34 smithi083 ceph-mon[24755]: pgmap v280: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.9 MiB/s wr, 749 op/s 2024-03-11T20:37:34.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:34 smithi204 ceph-mon[27258]: pgmap v280: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.9 MiB/s wr, 749 op/s 2024-03-11T20:37:35.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 50450 18.55 MB/sec warmup 78 sec latency 34.616 ms 2024-03-11T20:37:36.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 51280 18.53 MB/sec warmup 79 sec latency 19.354 ms 2024-03-11T20:37:36.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:36 smithi083 ceph-mon[24755]: pgmap v281: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-11T20:37:36.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:36 smithi204 ceph-mon[27258]: pgmap v281: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-11T20:37:37.008 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52153 18.53 MB/sec warmup 80 sec latency 30.741 ms 2024-03-11T20:37:38.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52861 18.66 MB/sec warmup 81 sec latency 22.954 ms 2024-03-11T20:37:38.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:38 smithi083 ceph-mon[24755]: pgmap v282: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-03-11T20:37:38.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:38 smithi204 ceph-mon[27258]: pgmap v282: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-03-11T20:37:39.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53417 18.72 MB/sec warmup 82 sec latency 48.170 ms 2024-03-11T20:37:40.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53936 18.56 MB/sec warmup 83 sec latency 22.114 ms 2024-03-11T20:37:40.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:40 smithi083 ceph-mon[24755]: pgmap v283: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-11T20:37:40.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:40 smithi204 ceph-mon[27258]: pgmap v283: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-03-11T20:37:41.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 54446 18.47 MB/sec warmup 84 sec latency 78.673 ms 2024-03-11T20:37:42.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55169 18.44 MB/sec warmup 85 sec latency 38.130 ms 2024-03-11T20:37:42.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:42 smithi083 ceph-mon[24755]: pgmap v284: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-03-11T20:37:42.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:42 smithi204 ceph-mon[27258]: pgmap v284: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-03-11T20:37:43.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55775 18.37 MB/sec warmup 86 sec latency 34.658 ms 2024-03-11T20:37:44.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 56556 18.54 MB/sec warmup 87 sec latency 26.946 ms 2024-03-11T20:37:44.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:44 smithi083 ceph-mon[24755]: pgmap v285: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 804 op/s 2024-03-11T20:37:44.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:44 smithi204 ceph-mon[27258]: pgmap v285: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 804 op/s 2024-03-11T20:37:45.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57004 18.54 MB/sec warmup 88 sec latency 23.584 ms 2024-03-11T20:37:46.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57429 18.38 MB/sec warmup 89 sec latency 29.936 ms 2024-03-11T20:37:46.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:46 smithi083 ceph-mon[24755]: pgmap v286: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 961 op/s 2024-03-11T20:37:46.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:46 smithi204 ceph-mon[27258]: pgmap v286: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 961 op/s 2024-03-11T20:37:47.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57858 18.28 MB/sec warmup 90 sec latency 51.452 ms 2024-03-11T20:37:48.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 58449 18.22 MB/sec warmup 91 sec latency 35.696 ms 2024-03-11T20:37:48.280 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:48 smithi083 ceph-mon[24755]: pgmap v287: 97 pgs: 97 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.5 MiB/s wr, 824 op/s 2024-03-11T20:37:48.425 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:48 smithi204 ceph-mon[27258]: pgmap v287: 97 pgs: 97 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.5 MiB/s wr, 824 op/s 2024-03-11T20:37:49.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59059 18.13 MB/sec warmup 92 sec latency 38.836 ms 2024-03-11T20:37:50.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59931 18.29 MB/sec warmup 93 sec latency 18.376 ms 2024-03-11T20:37:50.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:50 smithi083 ceph-mon[24755]: pgmap v288: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-03-11T20:37:50.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:50 smithi204 ceph-mon[27258]: pgmap v288: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-03-11T20:37:51.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 60648 18.39 MB/sec warmup 94 sec latency 23.354 ms 2024-03-11T20:37:52.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61249 18.27 MB/sec warmup 95 sec latency 39.407 ms 2024-03-11T20:37:52.818 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:37:52.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:52 smithi083 ceph-mon[24755]: pgmap v289: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 909 op/s 2024-03-11T20:37:52.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:52 smithi204 ceph-mon[27258]: pgmap v289: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 909 op/s 2024-03-11T20:37:53.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61693 18.21 MB/sec warmup 96 sec latency 77.553 ms 2024-03-11T20:37:53.783 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 99s ago 8m 22.6M - ba2b418f427c fc89fa4d50e8 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 99s ago 8m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (7m) 101s ago 7m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 99s ago 7m 51.9M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (2m) 99s ago 2m 13.6M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (2m) 101s ago 2m 14.4M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (9m) 99s ago 9m 462M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (7m) 101s ago 7m 412M - 17.2.0 e1d6a67b021e fac7917efddd 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 99s ago 9m 47.6M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (7m) 101s ago 7m 37.5M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (110s) 99s ago 109s 58.2M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (104s) 101s ago 103s 46.7M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:37:53.784 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 99s ago 7m 18.8M - 1dbe0e931976 8913d06e5699 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (7m) 101s ago 7m 17.4M - 1dbe0e931976 52a0feb54490 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 99s ago 6m 43.3M 3216M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 99s ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 99s ago 5m 39.1M 3216M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 99s ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (4m) 101s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (4m) 101s ago 4m 42.2M 3984M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (4m) 101s ago 4m 38.5M 3984M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (3m) 101s ago 3m 39.8M 3984M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:37:53.785 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 99s ago 6m 63.5M - 514e6a882f6e ed14b7fa7204 2024-03-11T20:37:54.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62329 18.14 MB/sec warmup 97 sec latency 34.754 ms 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:54.341 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:37:54.342 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:37:54.366 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: Upgrade: Updating mgr.smithi204.qknpir 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: pgmap v290: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 728 op/s 2024-03-11T20:37:54.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:54 smithi083 ceph-mon[24755]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: Upgrade: Updating mgr.smithi204.qknpir 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: pgmap v290: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 728 op/s 2024-03-11T20:37:54.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:54 smithi204 ceph-mon[27258]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "0/23 daemons upgraded", 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mgr daemons" 2024-03-11T20:37:54.767 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:37:55.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62803 18.06 MB/sec warmup 98 sec latency 28.478 ms 2024-03-11T20:37:55.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:55 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/776706454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:37:55.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:55 smithi204 ceph-mon[27258]: from='client.24425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:55.302 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:37:55.371 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:55 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/776706454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:37:55.371 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:55 smithi083 ceph-mon[24755]: from='client.24425 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:37:56.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63360 18.10 MB/sec warmup 99 sec latency 44.864 ms 2024-03-11T20:37:56.152 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:56 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1667128536' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:37:56.152 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:56 smithi083 ceph-mon[24755]: pgmap v291: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.4 MiB/s wr, 954 op/s 2024-03-11T20:37:56.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:56 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1667128536' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:37:56.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:56 smithi204 ceph-mon[27258]: pgmap v291: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.4 MiB/s wr, 954 op/s 2024-03-11T20:37:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63963 18.17 MB/sec warmup 100 sec latency 22.480 ms 2024-03-11T20:37:58.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64353 18.08 MB/sec warmup 101 sec latency 40.213 ms 2024-03-11T20:37:58.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:57 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:58.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:57 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:37:58.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:57 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:37:58.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:57 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:37:58.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:37:57 smithi083 ceph-mon[24755]: pgmap v292: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 834 op/s 2024-03-11T20:37:58.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:57 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:37:58.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:57 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:37:58.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:57 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:37:58.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:57 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:37:58.378 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:37:57 smithi204 ceph-mon[27258]: pgmap v292: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 834 op/s 2024-03-11T20:37:59.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64653 17.92 MB/sec warmup 102 sec latency 107.606 ms 2024-03-11T20:38:00.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65113 17.87 MB/sec warmup 103 sec latency 71.478 ms 2024-03-11T20:38:00.883 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:00 smithi083 ceph-mon[24755]: pgmap v293: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 707 op/s 2024-03-11T20:38:00.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:00 smithi204 ceph-mon[27258]: pgmap v293: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 707 op/s 2024-03-11T20:38:01.009 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65845 17.82 MB/sec warmup 104 sec latency 55.361 ms 2024-03-11T20:38:02.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 66357 17.75 MB/sec warmup 105 sec latency 98.190 ms 2024-03-11T20:38:02.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:02 smithi083 ceph-mon[24755]: pgmap v294: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 908 op/s 2024-03-11T20:38:02.882 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:02 smithi204 ceph-mon[27258]: pgmap v294: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 908 op/s 2024-03-11T20:38:03.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67186 17.93 MB/sec warmup 106 sec latency 40.823 ms 2024-03-11T20:38:04.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67659 17.93 MB/sec warmup 107 sec latency 99.114 ms 2024-03-11T20:38:04.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:04 smithi083 ceph-mon[24755]: pgmap v295: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 673 op/s 2024-03-11T20:38:04.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:04 smithi204 ceph-mon[27258]: pgmap v295: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 673 op/s 2024-03-11T20:38:05.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67989 17.79 MB/sec warmup 108 sec latency 124.339 ms 2024-03-11T20:38:05.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:38:05.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:38:05.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:38:05.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:38:05.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:38:05.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:05 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:38:05.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:05 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:38:06.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68379 17.69 MB/sec warmup 109 sec latency 69.059 ms 2024-03-11T20:38:06.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:06 smithi083 ceph-mon[24755]: pgmap v296: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-03-11T20:38:06.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:06 smithi083 ceph-mon[24755]: mgrmap e22: smithi083.jyxogi(active, since 7m), standbys: smithi204.qknpir 2024-03-11T20:38:06.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:06 smithi204 ceph-mon[27258]: pgmap v296: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-03-11T20:38:06.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:06 smithi204 ceph-mon[27258]: mgrmap e22: smithi083.jyxogi(active, since 7m), standbys: smithi204.qknpir 2024-03-11T20:38:07.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68920 17.67 MB/sec warmup 110 sec latency 56.437 ms 2024-03-11T20:38:08.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 69568 17.59 MB/sec warmup 111 sec latency 47.678 ms 2024-03-11T20:38:08.286 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:08 smithi083 ceph-mon[24755]: pgmap v297: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 763 op/s 2024-03-11T20:38:08.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:08 smithi204 ceph-mon[27258]: pgmap v297: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 763 op/s 2024-03-11T20:38:09.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 70223 17.65 MB/sec warmup 112 sec latency 32.359 ms 2024-03-11T20:38:09.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:09.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:09 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:09.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:09.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:09 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:10.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 70930 17.74 MB/sec warmup 113 sec latency 32.733 ms 2024-03-11T20:38:10.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:10 smithi083 ceph-mon[24755]: pgmap v298: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.2 MiB/s wr, 734 op/s 2024-03-11T20:38:10.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:10 smithi204 ceph-mon[27258]: pgmap v298: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.2 MiB/s wr, 734 op/s 2024-03-11T20:38:11.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71417 17.68 MB/sec warmup 114 sec latency 19.751 ms 2024-03-11T20:38:12.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71960 17.60 MB/sec warmup 115 sec latency 30.124 ms 2024-03-11T20:38:12.839 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:12 smithi083 ceph-mon[24755]: pgmap v299: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 1.06k op/s 2024-03-11T20:38:12.919 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:12 smithi204 ceph-mon[27258]: pgmap v299: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 1.06k op/s 2024-03-11T20:38:13.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72564 17.59 MB/sec warmup 116 sec latency 30.752 ms 2024-03-11T20:38:13.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:13 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:13.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:13 smithi204 ceph-mon[27258]: pgmap v300: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-03-11T20:38:13.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:13 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:14.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 73234 17.53 MB/sec warmup 117 sec latency 29.908 ms 2024-03-11T20:38:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:13 smithi083 ceph-mon[24755]: pgmap v300: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-03-11T20:38:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:13 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:15.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 73879 17.58 MB/sec warmup 118 sec latency 25.961 ms 2024-03-11T20:38:16.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 74497 17.65 MB/sec warmup 119 sec latency 27.288 ms 2024-03-11T20:38:16.826 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:16 smithi204 ceph-mon[27258]: pgmap v301: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.7 MiB/s wr, 981 op/s 2024-03-11T20:38:16.879 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:16 smithi083 ceph-mon[24755]: pgmap v301: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.7 MiB/s wr, 981 op/s 2024-03-11T20:38:18.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75508 8.81 MB/sec execute 1 sec latency 29.719 ms 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi083.jyxogi) 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: Failing over to other MGR 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr fail", "who": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: osdmap e71: 8 total, 8 up, 8 in 2024-03-11T20:38:18.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:18 smithi083 ceph-mon[24755]: pgmap v303: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 802 op/s 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi083.jyxogi) 2024-03-11T20:38:18.585 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: Failing over to other MGR 2024-03-11T20:38:18.586 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr fail", "who": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:38:18.586 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: osdmap e71: 8 total, 8 up, 8 in 2024-03-11T20:38:18.586 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:18 smithi204 ceph-mon[27258]: pgmap v303: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 802 op/s 2024-03-11T20:38:19.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75938 11.21 MB/sec execute 2 sec latency 56.219 ms 2024-03-11T20:38:19.400 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:38:19.400 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:38:19.400 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:38:19.400 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:38:19.400 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "mgr fail", "who": "smithi083.jyxogi"}]': finished 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: mgrmap e23: smithi204.qknpir(active, starting, since 0.991605s) 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: Manager daemon smithi204.qknpir is now available 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:19.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:19.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:19.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:38:19.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.14206 172.21.15.83:0/2685539936' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "mgr fail", "who": "smithi083.jyxogi"}]': finished 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: mgrmap e23: smithi204.qknpir(active, starting, since 0.991605s) 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:38:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: Manager daemon smithi204.qknpir is now available 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi204.qknpir/trash_purge_schedule"}]: dispatch 2024-03-11T20:38:19.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:19.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:20.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 76501 8.76 MB/sec execute 3 sec latency 34.671 ms 2024-03-11T20:38:20.413 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: Deploying cephadm binary to smithi204 2024-03-11T20:38:20.413 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: Deploying cephadm binary to smithi083 2024-03-11T20:38:20.413 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: mgrmap e24: smithi204.qknpir(active, since 2s) 2024-03-11T20:38:20.414 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:20.414 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: [11/Mar/2024:20:38:19] ENGINE Bus STARTING 2024-03-11T20:38:20.414 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: [11/Mar/2024:20:38:19] ENGINE Serving on http://172.21.15.204:8765 2024-03-11T20:38:20.414 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: [11/Mar/2024:20:38:19] ENGINE Serving on https://172.21.15.204:7150 2024-03-11T20:38:20.414 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:20 smithi083 ceph-mon[24755]: [11/Mar/2024:20:38:19] ENGINE Bus STARTED 2024-03-11T20:38:20.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: Deploying cephadm binary to smithi204 2024-03-11T20:38:20.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: Deploying cephadm binary to smithi083 2024-03-11T20:38:20.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: mgrmap e24: smithi204.qknpir(active, since 2s) 2024-03-11T20:38:20.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:20.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: [11/Mar/2024:20:38:19] ENGINE Bus STARTING 2024-03-11T20:38:20.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: [11/Mar/2024:20:38:19] ENGINE Serving on http://172.21.15.204:8765 2024-03-11T20:38:20.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: [11/Mar/2024:20:38:19] ENGINE Serving on https://172.21.15.204:7150 2024-03-11T20:38:20.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:20 smithi204 ceph-mon[27258]: [11/Mar/2024:20:38:19] ENGINE Bus STARTED 2024-03-11T20:38:21.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77195 12.31 MB/sec execute 4 sec latency 25.776 ms 2024-03-11T20:38:21.421 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:21 smithi083 ceph-mon[24755]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:21.421 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:21 smithi083 ceph-mon[24755]: mgrmap e25: smithi204.qknpir(active, since 3s) 2024-03-11T20:38:21.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:21 smithi204 ceph-mon[27258]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:21.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:21 smithi204 ceph-mon[27258]: mgrmap e25: smithi204.qknpir(active, since 3s) 2024-03-11T20:38:22.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77856 15.25 MB/sec execute 5 sec latency 32.316 ms 2024-03-11T20:38:23.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78223 14.95 MB/sec execute 6 sec latency 63.465 ms 2024-03-11T20:38:23.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:23 smithi083 ceph-mon[24755]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:23.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:23 smithi083 ceph-mon[24755]: mgrmap e26: smithi204.qknpir(active, since 5s) 2024-03-11T20:38:23.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:23 smithi204 ceph-mon[27258]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:23.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:23 smithi204 ceph-mon[27258]: mgrmap e26: smithi204.qknpir(active, since 5s) 2024-03-11T20:38:24.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78589 13.69 MB/sec execute 7 sec latency 40.441 ms 2024-03-11T20:38:25.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78969 12.82 MB/sec execute 8 sec latency 47.183 ms 2024-03-11T20:38:25.395 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:25 smithi083 ceph-mon[24755]: pgmap v6: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:25.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:25 smithi204 ceph-mon[27258]: pgmap v6: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-11T20:38:25.892 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:38:26.010 INFO:teuthology.orchestra.run.smithi083.stdout: 5 79303 12.39 MB/sec execute 9 sec latency 60.202 ms 2024-03-11T20:38:27.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 79741 11.93 MB/sec execute 10 sec latency 45.732 ms 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 2m ago 8m 22.6M - ba2b418f427c fc89fa4d50e8 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (8m) 14s ago 8m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 2m ago 8m 51.9M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:38:27.059 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (2m) 2m ago 2m 13.6M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (2m) 14s ago 2m 14.8M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (10m) 2m ago 10m 462M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (30s) 14s ago 8m 50.5M - 19.0.0-1979-g2b0101eb 4133f8bc41bb ce145ba7788d 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (10m) 2m ago 10m 47.6M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (7m) 14s ago 7m 38.8M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (2m) 2m ago 2m 58.2M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (2m) 14s ago 2m 72.9M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 2m ago 8m 18.8M - 1dbe0e931976 8913d06e5699 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (7m) 14s ago 7m 17.4M - 1dbe0e931976 52a0feb54490 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 2m ago 7m 43.3M 3216M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 2m ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 2m ago 6m 39.1M 3216M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 2m ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (5m) 14s ago 5m 233M 3984M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:38:27.060 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (4m) 14s ago 4m 273M 3984M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:38:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (4m) 14s ago 4m 244M 3984M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:38:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (4m) 14s ago 4m 334M 3984M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:38:27.061 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 2m ago 7m 63.5M - 514e6a882f6e ed14b7fa7204 2024-03-11T20:38:27.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:27.438 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: pgmap v7: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.9 MiB/s wr, 530 op/s 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/crt"}]: dispatch 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: Standby manager daemon smithi083.jyxogi started 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/key"}]: dispatch 2024-03-11T20:38:27.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:27 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:38:27.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:27.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: pgmap v7: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.9 MiB/s wr, 530 op/s 2024-03-11T20:38:27.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:27.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/crt"}]: dispatch 2024-03-11T20:38:27.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:38:27.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: Standby manager daemon smithi083.jyxogi started 2024-03-11T20:38:27.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/key"}]: dispatch 2024-03-11T20:38:27.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:27 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1585342019' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:38:27.701 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:38:27.701 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:38:27.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-11T20:38:27.703 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:38:27.703 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:38:27.703 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:38:28.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80273 11.81 MB/sec execute 11 sec latency 33.670 ms 2024-03-11T20:38:28.159 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "1/23 daemons upgraded", 2024-03-11T20:38:28.164 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-03-11T20:38:28.165 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:38:28.165 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:38:28.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:28 smithi083 ceph-mon[24755]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:28.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:28 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3687908378' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:38:28.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:28 smithi083 ceph-mon[24755]: mgrmap e27: smithi204.qknpir(active, since 10s), standbys: smithi083.jyxogi 2024-03-11T20:38:28.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:28 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:38:28.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:28 smithi204 ceph-mon[27258]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:28.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:28 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3687908378' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:38:28.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:28 smithi204 ceph-mon[27258]: mgrmap e27: smithi204.qknpir(active, since 10s), standbys: smithi083.jyxogi 2024-03-11T20:38:28.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:28 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:38:28.681 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:38:29.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80759 12.16 MB/sec execute 12 sec latency 48.449 ms 2024-03-11T20:38:29.496 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:29 smithi083 ceph-mon[24755]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:29.496 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:29 smithi083 ceph-mon[24755]: pgmap v8: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.6 MiB/s wr, 412 op/s 2024-03-11T20:38:29.497 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:29 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3609285819' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:38:29.497 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:29 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:29.497 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:29 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:29.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:29 smithi204 ceph-mon[27258]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:38:29.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:29 smithi204 ceph-mon[27258]: pgmap v8: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.6 MiB/s wr, 412 op/s 2024-03-11T20:38:29.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:29 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3609285819' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:38:29.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:29 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:29.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:29 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:30.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81229 12.38 MB/sec execute 13 sec latency 32.574 ms 2024-03-11T20:38:31.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81773 13.24 MB/sec execute 14 sec latency 26.683 ms 2024-03-11T20:38:31.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:31 smithi083 ceph-mon[24755]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-03-11T20:38:31.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:31 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:31.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:31 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:31.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:31 smithi204 ceph-mon[27258]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-03-11T20:38:31.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:31 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:31.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:31 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:32.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 82239 12.87 MB/sec execute 15 sec latency 60.304 ms 2024-03-11T20:38:33.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 82592 12.47 MB/sec execute 16 sec latency 33.711 ms 2024-03-11T20:38:33.288 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:33 smithi083 ceph-mon[24755]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.9 MiB/s wr, 579 op/s 2024-03-11T20:38:33.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:33 smithi204 ceph-mon[27258]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.9 MiB/s wr, 579 op/s 2024-03-11T20:38:34.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 83063 12.57 MB/sec execute 17 sec latency 48.680 ms 2024-03-11T20:38:34.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:34 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:34.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:34 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:34.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:34 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:34.293 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:34 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:34.294 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:34 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:34.294 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:34 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:35.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 83759 12.43 MB/sec execute 18 sec latency 37.056 ms 2024-03-11T20:38:35.298 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:35 smithi083 ceph-mon[24755]: pgmap v11: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.9 MiB/s wr, 579 op/s 2024-03-11T20:38:35.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:35 smithi204 ceph-mon[27258]: pgmap v11: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.9 MiB/s wr, 579 op/s 2024-03-11T20:38:36.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 84406 12.94 MB/sec execute 19 sec latency 46.936 ms 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi204 to 2704M 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: Adjusting osd_memory_target on smithi083 to 1936M 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:38:36.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:36 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:38:36.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi204 to 2704M 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: Adjusting osd_memory_target on smithi083 to 1936M 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:38:36.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:36 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:38:37.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85015 13.53 MB/sec execute 20 sec latency 54.007 ms 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:37 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:38:37.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:37.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:37 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:38.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85451 13.50 MB/sec execute 21 sec latency 32.316 ms 2024-03-11T20:38:38.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:38 smithi083 ceph-mon[24755]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-03-11T20:38:38.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:38 smithi083 ceph-mon[24755]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-03-11T20:38:39.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85906 13.18 MB/sec execute 22 sec latency 30.387 ms 2024-03-11T20:38:39.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:38 smithi204 ceph-mon[27258]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-03-11T20:38:39.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:38 smithi204 ceph-mon[27258]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-03-11T20:38:39.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:39 smithi083 ceph-mon[24755]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.3 MiB/s wr, 563 op/s 2024-03-11T20:38:39.999 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:39 smithi204 ceph-mon[27258]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.3 MiB/s wr, 563 op/s 2024-03-11T20:38:40.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 86312 13.02 MB/sec execute 23 sec latency 55.678 ms 2024-03-11T20:38:41.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 86951 13.01 MB/sec execute 24 sec latency 27.092 ms 2024-03-11T20:38:41.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:41 smithi083 ceph-mon[24755]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 888 op/s 2024-03-11T20:38:42.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 87529 13.03 MB/sec execute 25 sec latency 43.731 ms 2024-03-11T20:38:42.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:41 smithi204 ceph-mon[27258]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 888 op/s 2024-03-11T20:38:43.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 88113 13.36 MB/sec execute 26 sec latency 24.920 ms 2024-03-11T20:38:43.183 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:43 smithi083 ceph-mon[24755]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 617 op/s 2024-03-11T20:38:43.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:43 smithi204 ceph-mon[27258]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 617 op/s 2024-03-11T20:38:44.011 INFO:teuthology.orchestra.run.smithi083.stdout: 5 88734 13.80 MB/sec execute 27 sec latency 49.937 ms 2024-03-11T20:38:45.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89130 13.66 MB/sec execute 28 sec latency 37.614 ms 2024-03-11T20:38:45.480 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:45 smithi083 ceph-mon[24755]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 617 op/s 2024-03-11T20:38:45.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:45 smithi204 ceph-mon[27258]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 617 op/s 2024-03-11T20:38:46.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89516 13.35 MB/sec execute 29 sec latency 32.608 ms 2024-03-11T20:38:47.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89901 13.18 MB/sec execute 30 sec latency 34.442 ms 2024-03-11T20:38:47.505 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:47 smithi083 ceph-mon[24755]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 945 op/s 2024-03-11T20:38:47.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:47 smithi204 ceph-mon[27258]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 945 op/s 2024-03-11T20:38:48.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 90516 13.18 MB/sec execute 31 sec latency 66.681 ms 2024-03-11T20:38:49.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 91107 13.20 MB/sec execute 32 sec latency 33.639 ms 2024-03-11T20:38:49.522 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:49 smithi083 ceph-mon[24755]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-03-11T20:38:49.522 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:49 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:49.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:49 smithi204 ceph-mon[27258]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-03-11T20:38:49.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:49 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:38:50.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 91748 13.52 MB/sec execute 33 sec latency 30.541 ms 2024-03-11T20:38:51.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 92328 13.80 MB/sec execute 34 sec latency 25.727 ms 2024-03-11T20:38:51.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:51 smithi083 ceph-mon[24755]: pgmap v19: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 962 op/s 2024-03-11T20:38:51.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:51 smithi204 ceph-mon[27258]: pgmap v19: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 962 op/s 2024-03-11T20:38:52.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 92791 13.69 MB/sec execute 35 sec latency 28.561 ms 2024-03-11T20:38:53.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93260 13.54 MB/sec execute 36 sec latency 46.460 ms 2024-03-11T20:38:53.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:53 smithi083 ceph-mon[24755]: pgmap v20: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-03-11T20:38:53.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:53 smithi204 ceph-mon[27258]: pgmap v20: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-03-11T20:38:54.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93773 13.54 MB/sec execute 37 sec latency 41.876 ms 2024-03-11T20:38:55.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 94276 13.32 MB/sec execute 38 sec latency 39.139 ms 2024-03-11T20:38:55.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:55 smithi083 ceph-mon[24755]: pgmap v21: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-03-11T20:38:55.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:55 smithi204 ceph-mon[27258]: pgmap v21: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-03-11T20:38:56.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 94714 13.32 MB/sec execute 39 sec latency 39.740 ms 2024-03-11T20:38:57.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 95227 13.53 MB/sec execute 40 sec latency 34.540 ms 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-03-11T20:38:57.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:56 smithi083 ceph-mon[24755]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 954 op/s 2024-03-11T20:38:57.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-03-11T20:38:57.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:56 smithi204 ceph-mon[27258]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 954 op/s 2024-03-11T20:38:58.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 95718 13.69 MB/sec execute 41 sec latency 54.214 ms 2024-03-11T20:38:59.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96097 13.56 MB/sec execute 42 sec latency 23.722 ms 2024-03-11T20:38:59.268 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:38:59.431 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:38:59 smithi083 ceph-mon[24755]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 625 op/s 2024-03-11T20:38:59.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:38:59 smithi204 ceph-mon[27258]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 625 op/s 2024-03-11T20:39:00.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96492 13.42 MB/sec execute 43 sec latency 44.943 ms 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 starting - - - - 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 26s ago 9m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (8m) 31s ago 8m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (7m) 26s ago 9m 52.0M - 8.3.5 dad864ee21e9 f735a2ea585f 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (3m) 26s ago 3m 36.7M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (3m) 31s ago 3m 15.2M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (10m) 26s ago 10m 336M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (63s) 31s ago 8m 490M - 19.0.0-1979-g2b0101eb 4133f8bc41bb ce145ba7788d 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (10m) 26s ago 10m 48.7M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (8m) 31s ago 8m 40.2M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:39:00.399 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (2m) 26s ago 2m 253M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (2m) 31s ago 2m 75.3M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 26s ago 8m 17.4M - 1.3.1 1dbe0e931976 8913d06e5699 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (8m) 31s ago 8m 17.4M - 1.3.1 1dbe0e931976 52a0feb54490 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 26s ago 7m 329M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 26s ago 7m 255M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 26s ago 6m 194M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 26s ago 6m 272M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (5m) 31s ago 5m 295M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (5m) 31s ago 5m 325M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (5m) 31s ago 5m 267M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (4m) 31s ago 4m 402M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:39:00.400 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 26s ago 8m 64.9M - 2.33.4 514e6a882f6e ed14b7fa7204 2024-03-11T20:39:00.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:00 smithi083 ceph-mon[24755]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:00.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:00 smithi083 ceph-mon[24755]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:00.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:00 smithi204 ceph-mon[27258]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:00.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:00 smithi204 ceph-mon[27258]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:01.012 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96835 13.29 MB/sec execute 44 sec latency 40.422 ms 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:01.095 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:39:01.096 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:39:01.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:01 smithi083 ceph-mon[24755]: pgmap v24: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 890 op/s 2024-03-11T20:39:01.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:01 smithi083 ceph-mon[24755]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:01.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:01 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/370415778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:39:01.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:01 smithi204 ceph-mon[27258]: pgmap v24: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 890 op/s 2024-03-11T20:39:01.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:01 smithi204 ceph-mon[27258]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:01.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:01 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/370415778' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "1/23 daemons upgraded", 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:39:01.713 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:39:02.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 97250 13.25 MB/sec execute 45 sec latency 60.404 ms 2024-03-11T20:39:02.445 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:39:02.498 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:02 smithi083 ceph-mon[24755]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:02.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:02 smithi204 ceph-mon[27258]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:03.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 97662 13.05 MB/sec execute 46 sec latency 40.894 ms 2024-03-11T20:39:03.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:03 smithi083 ceph-mon[24755]: pgmap v25: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 581 op/s 2024-03-11T20:39:03.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:03 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/2752885663' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:39:03.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:03 smithi204 ceph-mon[27258]: pgmap v25: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 581 op/s 2024-03-11T20:39:03.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:03 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/2752885663' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:39:04.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98152 13.04 MB/sec execute 47 sec latency 83.934 ms 2024-03-11T20:39:04.518 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:04 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:04.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:04 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:05.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98527 12.97 MB/sec execute 48 sec latency 34.659 ms 2024-03-11T20:39:05.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:05 smithi083 ceph-mon[24755]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 581 op/s 2024-03-11T20:39:05.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:05 smithi204 ceph-mon[27258]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 581 op/s 2024-03-11T20:39:06.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98960 13.06 MB/sec execute 49 sec latency 35.577 ms 2024-03-11T20:39:07.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 99323 13.17 MB/sec execute 50 sec latency 46.591 ms 2024-03-11T20:39:07.593 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:07 smithi083 ceph-mon[24755]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 836 op/s 2024-03-11T20:39:07.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:07 smithi204 ceph-mon[27258]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 836 op/s 2024-03-11T20:39:08.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 99672 13.10 MB/sec execute 51 sec latency 56.528 ms 2024-03-11T20:39:09.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 99986 12.92 MB/sec execute 52 sec latency 57.495 ms 2024-03-11T20:39:09.640 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:09 smithi083 ceph-mon[24755]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-03-11T20:39:09.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:09 smithi204 ceph-mon[27258]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-03-11T20:39:10.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 100333 12.80 MB/sec execute 53 sec latency 45.996 ms 2024-03-11T20:39:11.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 100646 12.65 MB/sec execute 54 sec latency 56.911 ms 2024-03-11T20:39:11.519 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:11 smithi083 ceph-mon[24755]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.9 MiB/s wr, 740 op/s 2024-03-11T20:39:11.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:11 smithi204 ceph-mon[27258]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.9 MiB/s wr, 740 op/s 2024-03-11T20:39:12.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101027 12.63 MB/sec execute 55 sec latency 52.817 ms 2024-03-11T20:39:13.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101508 12.53 MB/sec execute 56 sec latency 36.730 ms 2024-03-11T20:39:13.347 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:13 smithi083 ceph-mon[24755]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-03-11T20:39:13.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:13 smithi204 ceph-mon[27258]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-03-11T20:39:14.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101864 12.49 MB/sec execute 57 sec latency 32.175 ms 2024-03-11T20:39:15.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 102377 12.64 MB/sec execute 58 sec latency 36.351 ms 2024-03-11T20:39:15.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:15 smithi083 ceph-mon[24755]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-03-11T20:39:15.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:15 smithi204 ceph-mon[27258]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-03-11T20:39:16.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 102826 12.77 MB/sec execute 59 sec latency 38.798 ms 2024-03-11T20:39:17.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103156 12.68 MB/sec execute 60 sec latency 41.174 ms 2024-03-11T20:39:17.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:17 smithi083 ceph-mon[24755]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 717 op/s 2024-03-11T20:39:17.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:17 smithi204 ceph-mon[27258]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 717 op/s 2024-03-11T20:39:18.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103480 12.57 MB/sec execute 61 sec latency 36.752 ms 2024-03-11T20:39:19.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103833 12.48 MB/sec execute 62 sec latency 35.601 ms 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: Reconfiguring node-exporter.smithi083 (dependencies changed)... 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: Reconfiguring daemon node-exporter.smithi083 on smithi083 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 462 op/s 2024-03-11T20:39:19.441 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:19.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: Reconfiguring node-exporter.smithi083 (dependencies changed)... 2024-03-11T20:39:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: Reconfiguring daemon node-exporter.smithi083 on smithi083 2024-03-11T20:39:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 462 op/s 2024-03-11T20:39:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:20.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 104280 12.41 MB/sec execute 63 sec latency 30.149 ms 2024-03-11T20:39:21.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 104773 12.37 MB/sec execute 64 sec latency 64.220 ms 2024-03-11T20:39:21.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:21 smithi083 ceph-mon[24755]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-03-11T20:39:21.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:21 smithi204 ceph-mon[27258]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 700 op/s 2024-03-11T20:39:22.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 105254 12.37 MB/sec execute 65 sec latency 78.910 ms 2024-03-11T20:39:23.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 105753 12.45 MB/sec execute 66 sec latency 37.961 ms 2024-03-11T20:39:23.506 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:23 smithi083 ceph-mon[24755]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-03-11T20:39:23.506 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:23 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:23.506 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:23 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:23.506 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:23 smithi083 ceph-mon[24755]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-03-11T20:39:23.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:23 smithi083 ceph-mon[24755]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-03-11T20:39:23.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:23 smithi204 ceph-mon[27258]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-03-11T20:39:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:23 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:23 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:23 smithi204 ceph-mon[27258]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-03-11T20:39:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:23 smithi204 ceph-mon[27258]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-03-11T20:39:24.013 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106153 12.48 MB/sec execute 67 sec latency 42.110 ms 2024-03-11T20:39:25.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106536 12.54 MB/sec execute 68 sec latency 38.776 ms 2024-03-11T20:39:25.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:25 smithi083 ceph-mon[24755]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-03-11T20:39:25.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:25 smithi204 ceph-mon[27258]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-03-11T20:39:26.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106803 12.47 MB/sec execute 69 sec latency 46.177 ms 2024-03-11T20:39:27.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 107132 12.34 MB/sec execute 70 sec latency 70.318 ms 2024-03-11T20:39:27.562 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:27 smithi083 ceph-mon[24755]: pgmap v37: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-03-11T20:39:27.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:27 smithi204 ceph-mon[27258]: pgmap v37: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-03-11T20:39:28.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 107638 12.30 MB/sec execute 71 sec latency 37.978 ms 2024-03-11T20:39:29.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 108325 12.32 MB/sec execute 72 sec latency 40.440 ms 2024-03-11T20:39:29.573 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:29 smithi083 ceph-mon[24755]: pgmap v38: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 509 op/s 2024-03-11T20:39:29.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:29 smithi204 ceph-mon[27258]: pgmap v38: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 509 op/s 2024-03-11T20:39:30.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 108992 12.37 MB/sec execute 73 sec latency 22.541 ms 2024-03-11T20:39:31.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 109708 12.58 MB/sec execute 74 sec latency 23.274 ms 2024-03-11T20:39:31.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:31 smithi083 ceph-mon[24755]: pgmap v39: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 791 op/s 2024-03-11T20:39:31.926 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:31 smithi204 ceph-mon[27258]: pgmap v39: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 791 op/s 2024-03-11T20:39:32.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110213 12.69 MB/sec execute 75 sec latency 55.422 ms 2024-03-11T20:39:32.967 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:39:33.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110566 12.61 MB/sec execute 76 sec latency 44.315 ms 2024-03-11T20:39:33.372 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:33 smithi083 ceph-mon[24755]: pgmap v40: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 553 op/s 2024-03-11T20:39:33.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:33 smithi204 ceph-mon[27258]: pgmap v40: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 553 op/s 2024-03-11T20:39:33.776 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 starting - - - - 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 59s ago 9m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (9m) 64s ago 9m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 starting - - - - 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (3m) 59s ago 3m 36.7M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (3m) 64s ago 3m 15.2M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:9283 running (11m) 59s ago 11m 336M - 17.2.0 e1d6a67b021e e245c05d5751 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (97s) 64s ago 9m 490M - 19.0.0-1979-g2b0101eb 4133f8bc41bb ce145ba7788d 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (11m) 59s ago 11m 48.7M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (9m) 64s ago 9m 40.2M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (3m) 59s ago 3m 253M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (3m) 64s ago 3m 75.3M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:39:33.777 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 starting - - - - 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (8m) 64s ago 8m 17.4M - 1.3.1 1dbe0e931976 52a0feb54490 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 59s ago 8m 329M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 59s ago 7m 255M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 59s ago 7m 194M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 59s ago 6m 272M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (6m) 64s ago 6m 295M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (6m) 64s ago 6m 325M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (5m) 64s ago 5m 267M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (5m) 64s ago 5m 402M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:39:33.778 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 59s ago 8m 64.9M - 2.33.4 514e6a882f6e ed14b7fa7204 2024-03-11T20:39:34.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110874 12.49 MB/sec execute 77 sec latency 32.162 ms 2024-03-11T20:39:34.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:34 smithi083 ceph-mon[24755]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:34 smithi083 ceph-mon[24755]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:34 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:34.140 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:34 smithi083 ceph-mon[24755]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:34 smithi204 ceph-mon[27258]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:34 smithi204 ceph-mon[27258]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:34 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:34.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:34 smithi204 ceph-mon[27258]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:39:34.490 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:39:34.491 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "1/23 daemons upgraded", 2024-03-11T20:39:34.910 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-03-11T20:39:34.911 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:39:34.911 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:39:35.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 111187 12.42 MB/sec execute 78 sec latency 46.502 ms 2024-03-11T20:39:35.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:35 smithi083 ceph-mon[24755]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 553 op/s 2024-03-11T20:39:35.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:35 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3549706769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:39:35.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:35 smithi204 ceph-mon[27258]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 553 op/s 2024-03-11T20:39:35.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:35 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3549706769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:39:35.473 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:39:36.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 111621 12.42 MB/sec execute 79 sec latency 86.031 ms 2024-03-11T20:39:36.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:36 smithi083 ceph-mon[24755]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:36.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:36 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3195365469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:39:36.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:36 smithi204 ceph-mon[27258]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:39:36.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:36 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3195365469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:39:37.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 112018 12.30 MB/sec execute 80 sec latency 79.113 ms 2024-03-11T20:39:37.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:37 smithi083 ceph-mon[24755]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-03-11T20:39:37.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:37 smithi204 ceph-mon[27258]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-03-11T20:39:38.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 112418 12.31 MB/sec execute 81 sec latency 67.847 ms 2024-03-11T20:39:39.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 112853 12.34 MB/sec execute 82 sec latency 27.128 ms 2024-03-11T20:39:39.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:39 smithi083 ceph-mon[24755]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-03-11T20:39:39.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:39 smithi204 ceph-mon[27258]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-03-11T20:39:40.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 113419 12.49 MB/sec execute 83 sec latency 25.825 ms 2024-03-11T20:39:41.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 113752 12.46 MB/sec execute 84 sec latency 28.525 ms 2024-03-11T20:39:41.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:41 smithi083 ceph-mon[24755]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 821 op/s 2024-03-11T20:39:41.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:41 smithi204 ceph-mon[27258]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 821 op/s 2024-03-11T20:39:42.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114092 12.39 MB/sec execute 85 sec latency 40.054 ms 2024-03-11T20:39:43.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114404 12.29 MB/sec execute 86 sec latency 36.832 ms 2024-03-11T20:39:43.168 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:43 smithi083 ceph-mon[24755]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-03-11T20:39:43.426 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:43 smithi204 ceph-mon[27258]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-03-11T20:39:44.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114743 12.23 MB/sec execute 87 sec latency 37.238 ms 2024-03-11T20:39:45.014 INFO:teuthology.orchestra.run.smithi083.stdout: 5 115244 12.24 MB/sec execute 88 sec latency 37.595 ms 2024-03-11T20:39:45.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:45 smithi083 ceph-mon[24755]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-03-11T20:39:45.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:45 smithi204 ceph-mon[27258]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 540 op/s 2024-03-11T20:39:46.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 115746 12.20 MB/sec execute 89 sec latency 26.457 ms 2024-03-11T20:39:47.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116257 12.25 MB/sec execute 90 sec latency 41.004 ms 2024-03-11T20:39:47.480 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:47 smithi083 ceph-mon[24755]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 777 op/s 2024-03-11T20:39:47.676 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:47 smithi204 ceph-mon[27258]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 777 op/s 2024-03-11T20:39:48.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116804 12.33 MB/sec execute 91 sec latency 52.056 ms 2024-03-11T20:39:49.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 117324 12.41 MB/sec execute 92 sec latency 34.372 ms 2024-03-11T20:39:49.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 484 op/s 2024-03-11T20:39:49.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:49.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:49.512 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:49.512 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: Reconfiguring node-exporter.smithi204 (dependencies changed)... 2024-03-11T20:39:49.512 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:49 smithi083 ceph-mon[24755]: Reconfiguring daemon node-exporter.smithi204 on smithi204 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 484 op/s 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: Reconfiguring node-exporter.smithi204 (dependencies changed)... 2024-03-11T20:39:49.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:49 smithi204 ceph-mon[27258]: Reconfiguring daemon node-exporter.smithi204 on smithi204 2024-03-11T20:39:50.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 117710 12.37 MB/sec execute 93 sec latency 36.734 ms 2024-03-11T20:39:51.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118130 12.32 MB/sec execute 94 sec latency 40.722 ms 2024-03-11T20:39:51.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 778 op/s 2024-03-11T20:39:51.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: mgrmap e28: smithi204.qknpir(active, since 93s), standbys: smithi083.jyxogi 2024-03-11T20:39:51.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:51 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 778 op/s 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: mgrmap e28: smithi204.qknpir(active, since 93s), standbys: smithi083.jyxogi 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:51.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:51 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:39:52.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118519 12.29 MB/sec execute 95 sec latency 46.804 ms 2024-03-11T20:39:52.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:39:52.556 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:52 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:39:52.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:52 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:39:53.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118961 12.23 MB/sec execute 96 sec latency 39.155 ms 2024-03-11T20:39:53.569 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:53 smithi083 ceph-mon[24755]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-11T20:39:53.828 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:53 smithi204 ceph-mon[27258]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-11T20:39:54.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 119424 12.23 MB/sec execute 97 sec latency 40.786 ms 2024-03-11T20:39:55.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 120051 12.32 MB/sec execute 98 sec latency 55.162 ms 2024-03-11T20:39:55.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: Upgrade: Updating mgr.smithi083.jyxogi 2024-03-11T20:39:55.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:55.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:39:55.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:39:55.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:39:55.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:39:55.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: Deploying daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:39:55.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:55 smithi083 ceph-mon[24755]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-11T20:39:55.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: Upgrade: Updating mgr.smithi083.jyxogi 2024-03-11T20:39:55.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:39:55.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:39:55.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:39:55.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:39:55.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:39:55.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: Deploying daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:39:55.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:55 smithi204 ceph-mon[27258]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-03-11T20:39:56.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 120682 12.45 MB/sec execute 99 sec latency 33.089 ms 2024-03-11T20:39:57.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121168 12.45 MB/sec execute 100 sec latency 25.800 ms 2024-03-11T20:39:57.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:57 smithi083 ceph-mon[24755]: pgmap v52: 97 pgs: 97 active+clean; 325 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 807 op/s 2024-03-11T20:39:57.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:57 smithi204 ceph-mon[27258]: pgmap v52: 97 pgs: 97 active+clean; 325 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 807 op/s 2024-03-11T20:39:58.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121561 12.40 MB/sec execute 101 sec latency 47.503 ms 2024-03-11T20:39:59.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121977 12.34 MB/sec execute 102 sec latency 44.694 ms 2024-03-11T20:39:59.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:39:59 smithi083 ceph-mon[24755]: pgmap v53: 97 pgs: 97 active+clean; 325 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-03-11T20:39:59.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:39:59 smithi204 ceph-mon[27258]: pgmap v53: 97 pgs: 97 active+clean; 325 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-03-11T20:40:00.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122394 12.30 MB/sec execute 103 sec latency 42.215 ms 2024-03-11T20:40:00.677 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:00 smithi083 ceph-mon[24755]: overall HEALTH_OK 2024-03-11T20:40:00.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:00 smithi204 ceph-mon[27258]: overall HEALTH_OK 2024-03-11T20:40:01.015 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122975 12.33 MB/sec execute 104 sec latency 34.227 ms 2024-03-11T20:40:01.486 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:01 smithi083 ceph-mon[24755]: pgmap v54: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-03-11T20:40:01.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:01 smithi204 ceph-mon[27258]: pgmap v54: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-03-11T20:40:02.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 123615 12.42 MB/sec execute 105 sec latency 61.375 ms 2024-03-11T20:40:03.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124103 12.50 MB/sec execute 106 sec latency 114.404 ms 2024-03-11T20:40:03.249 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:03 smithi083 ceph-mon[24755]: pgmap v55: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 580 op/s 2024-03-11T20:40:03.249 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:03 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:03.249 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:03 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:03.249 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:03 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:03.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:03 smithi204 ceph-mon[27258]: pgmap v55: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 580 op/s 2024-03-11T20:40:03.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:03 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:03.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:03 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:03.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:03 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:04.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124453 12.49 MB/sec execute 107 sec latency 49.421 ms 2024-03-11T20:40:04.258 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:04 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:04.258 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:04 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:04.259 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:04 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:04.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:04 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:04.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:04 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:04.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:04 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:05.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124714 12.41 MB/sec execute 108 sec latency 70.152 ms 2024-03-11T20:40:05.278 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:05 smithi083 ceph-mon[24755]: pgmap v56: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 580 op/s 2024-03-11T20:40:05.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:05 smithi204 ceph-mon[27258]: pgmap v56: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 580 op/s 2024-03-11T20:40:05.984 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:40:06.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124970 12.33 MB/sec execute 109 sec latency 41.329 ms 2024-03-11T20:40:07.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 125331 12.27 MB/sec execute 110 sec latency 80.377 ms 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 starting - - - - 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (10m) 93s ago 10m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (9m) 0s ago 9m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 starting - - - - 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (4m) 93s ago 4m 36.7M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (4m) 0s ago 4m 15.7M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 starting - - - - 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (2m) 0s ago 9m 503M - 19.0.0-1979-g2b0101eb 4133f8bc41bb ce145ba7788d 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (11m) 93s ago 11m 48.7M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (9m) 0s ago 9m 43.7M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:40:07.023 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (4m) 93s ago 4m 253M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (3m) 0s ago 3m 83.2M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 starting - - - - 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (16s) 0s ago 9m 3548k - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 93s ago 8m 329M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 93s ago 8m 255M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 93s ago 7m 194M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 93s ago 7m 272M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (7m) 0s ago 7m 419M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (6m) 0s ago 6m 469M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (6m) 0s ago 6m 404M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (5m) 0s ago 5m 546M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:40:07.024 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 starting - - - - 2024-03-11T20:40:07.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:07 smithi083 ceph-mon[24755]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:07.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:07 smithi083 ceph-mon[24755]: pgmap v57: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 852 op/s 2024-03-11T20:40:07.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:07 smithi083 ceph-mon[24755]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:07.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:07 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:07.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:07 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:07.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:07 smithi204 ceph-mon[27258]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:07.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:07 smithi204 ceph-mon[27258]: pgmap v57: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 852 op/s 2024-03-11T20:40:07.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:07 smithi204 ceph-mon[27258]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:07.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:07 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:07.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:07 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:07.724 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:40:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:40:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:40:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:07.724 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:40:07.725 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:40:08.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 125885 12.27 MB/sec execute 111 sec latency 24.526 ms 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mgr daemons", 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:40:08.274 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:40:08.562 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:08 smithi083 ceph-mon[24755]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:08.562 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:08 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3584505829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:40:08.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:08 smithi204 ceph-mon[27258]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:08.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:08 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3584505829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:40:08.965 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:40:09.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 126296 12.25 MB/sec execute 112 sec latency 44.841 ms 2024-03-11T20:40:09.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:09 smithi083 ceph-mon[24755]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:09.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:09 smithi083 ceph-mon[24755]: pgmap v58: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-03-11T20:40:09.566 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:09 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/1009751162' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:40:09.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:09 smithi204 ceph-mon[27258]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:09.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:09 smithi204 ceph-mon[27258]: pgmap v58: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-03-11T20:40:09.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:09 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1009751162' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:40:10.016 INFO:teuthology.orchestra.run.smithi083.stdout: 5 126737 12.22 MB/sec execute 113 sec latency 29.462 ms 2024-03-11T20:40:11.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 127162 12.27 MB/sec execute 114 sec latency 32.393 ms 2024-03-11T20:40:11.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:11 smithi083 ceph-mon[24755]: pgmap v59: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 822 op/s 2024-03-11T20:40:11.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:11 smithi204 ceph-mon[27258]: pgmap v59: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 822 op/s 2024-03-11T20:40:12.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 127553 12.33 MB/sec execute 115 sec latency 51.813 ms 2024-03-11T20:40:13.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 127815 12.29 MB/sec execute 116 sec latency 40.744 ms 2024-03-11T20:40:13.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:13.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:13.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: Standby manager daemon smithi083.jyxogi restarted 2024-03-11T20:40:13.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: Standby manager daemon smithi083.jyxogi started 2024-03-11T20:40:13.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/crt"}]: dispatch 2024-03-11T20:40:13.389 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:40:13.389 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/key"}]: dispatch 2024-03-11T20:40:13.390 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:40:13.390 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:13 smithi083 ceph-mon[24755]: pgmap v60: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 518 op/s 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: Standby manager daemon smithi083.jyxogi restarted 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: Standby manager daemon smithi083.jyxogi started 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/crt"}]: dispatch 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:40:13.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.jyxogi/key"}]: dispatch 2024-03-11T20:40:13.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:40:13.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:13 smithi204 ceph-mon[27258]: pgmap v60: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 518 op/s 2024-03-11T20:40:14.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128070 12.25 MB/sec execute 117 sec latency 49.879 ms 2024-03-11T20:40:14.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:14 smithi083 ceph-mon[24755]: mgrmap e29: smithi204.qknpir(active, since 115s), standbys: smithi083.jyxogi 2024-03-11T20:40:14.401 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:14 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:14.402 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:14 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:14.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:14 smithi204 ceph-mon[27258]: mgrmap e29: smithi204.qknpir(active, since 115s), standbys: smithi083.jyxogi 2024-03-11T20:40:14.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:14 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:14.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:14 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:15.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128309 12.17 MB/sec execute 118 sec latency 48.777 ms 2024-03-11T20:40:15.158 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:15 smithi083 ceph-mon[24755]: pgmap v61: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 518 op/s 2024-03-11T20:40:15.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:15 smithi204 ceph-mon[27258]: pgmap v61: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 518 op/s 2024-03-11T20:40:16.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128571 12.10 MB/sec execute 119 sec latency 41.755 ms 2024-03-11T20:40:17.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128850 12.05 MB/sec execute 120 sec latency 81.202 ms 2024-03-11T20:40:17.501 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:17 smithi083 ceph-mon[24755]: pgmap v62: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 712 op/s 2024-03-11T20:40:17.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:17 smithi204 ceph-mon[27258]: pgmap v62: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 712 op/s 2024-03-11T20:40:18.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129192 12.01 MB/sec execute 121 sec latency 65.231 ms 2024-03-11T20:40:19.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129777 12.02 MB/sec execute 122 sec latency 25.153 ms 2024-03-11T20:40:19.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:19 smithi083 ceph-mon[24755]: pgmap v63: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-03-11T20:40:19.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:19 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:19 smithi204 ceph-mon[27258]: pgmap v63: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-03-11T20:40:19.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:19 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:20.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 130401 12.06 MB/sec execute 123 sec latency 33.091 ms 2024-03-11T20:40:20.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: Failing over to other MGR 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: osdmap e72: 8 total, 8 up, 8 in 2024-03-11T20:40:20.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:20 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:40:20.882 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.882 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.882 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:20.882 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: Upgrade: Need to upgrade myself (mgr.smithi204.qknpir) 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: Failing over to other MGR 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 172.21.15.204:0/3205770212' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: osdmap e72: 8 total, 8 up, 8 in 2024-03-11T20:40:20.883 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:20 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:40:21.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 130770 12.07 MB/sec execute 124 sec latency 38.754 ms 2024-03-11T20:40:21.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd='[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]': finished 2024-03-11T20:40:21.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: mgrmap e30: smithi083.jyxogi(active, starting, since 0.863222s), standbys: smithi204.qknpir 2024-03-11T20:40:21.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:40:21.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:40:21.614 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:40:21.615 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:40:21.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:21 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24431 ' entity='mgr.smithi204.qknpir' cmd='[{"prefix": "mgr fail", "who": "smithi204.qknpir"}]': finished 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: mgrmap e30: smithi083.jyxogi(active, starting, since 0.863222s), standbys: smithi204.qknpir 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:40:21.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:40:21.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:21 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:40:22.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131099 12.11 MB/sec execute 125 sec latency 72.572 ms 2024-03-11T20:40:22.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: [11/Mar/2024:20:40:21] ENGINE Bus STARTING 2024-03-11T20:40:22.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: [11/Mar/2024:20:40:21] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:40:22.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: mgrmap e31: smithi083.jyxogi(active, since 1.87541s), standbys: smithi204.qknpir 2024-03-11T20:40:22.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:22.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: [11/Mar/2024:20:40:21] ENGINE Serving on http://172.21.15.83:8765 2024-03-11T20:40:22.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: [11/Mar/2024:20:40:21] ENGINE Bus STARTED 2024-03-11T20:40:22.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:22.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:22 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:22.669 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: [11/Mar/2024:20:40:21] ENGINE Bus STARTING 2024-03-11T20:40:22.669 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: [11/Mar/2024:20:40:21] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: mgrmap e31: smithi083.jyxogi(active, since 1.87541s), standbys: smithi204.qknpir 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: [11/Mar/2024:20:40:21] ENGINE Serving on http://172.21.15.83:8765 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: [11/Mar/2024:20:40:21] ENGINE Bus STARTED 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:22.670 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:22 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:23.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131319 12.06 MB/sec execute 126 sec latency 50.516 ms 2024-03-11T20:40:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:23 smithi204 ceph-mon[27258]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:23.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:23 smithi204 ceph-mon[27258]: mgrmap e32: smithi083.jyxogi(active, since 3s), standbys: smithi204.qknpir 2024-03-11T20:40:23.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:23 smithi083 ceph-mon[24755]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:23.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:23 smithi083 ceph-mon[24755]: mgrmap e32: smithi083.jyxogi(active, since 3s), standbys: smithi204.qknpir 2024-03-11T20:40:24.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131577 12.03 MB/sec execute 127 sec latency 87.717 ms 2024-03-11T20:40:25.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131793 11.96 MB/sec execute 128 sec latency 56.044 ms 2024-03-11T20:40:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:25 smithi083 ceph-mon[24755]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:25 smithi083 ceph-mon[24755]: mgrmap e33: smithi083.jyxogi(active, since 4s), standbys: smithi204.qknpir 2024-03-11T20:40:25.914 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:25 smithi204 ceph-mon[27258]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:40:25.925 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:25 smithi204 ceph-mon[27258]: mgrmap e33: smithi083.jyxogi(active, since 4s), standbys: smithi204.qknpir 2024-03-11T20:40:26.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131996 11.89 MB/sec execute 129 sec latency 57.261 ms 2024-03-11T20:40:27.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 132246 11.85 MB/sec execute 130 sec latency 63.596 ms 2024-03-11T20:40:27.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:27 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:27.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:27 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:27.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:27 smithi083 ceph-mon[24755]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 703 KiB/s wr, 31 op/s 2024-03-11T20:40:27.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:27 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:27.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:27 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:27.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:27 smithi204 ceph-mon[27258]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 703 KiB/s wr, 31 op/s 2024-03-11T20:40:28.019 INFO:teuthology.orchestra.run.smithi083.stdout: 5 132506 11.79 MB/sec execute 131 sec latency 94.514 ms 2024-03-11T20:40:29.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 132994 11.77 MB/sec execute 132 sec latency 35.194 ms 2024-03-11T20:40:29.135 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:28 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:29.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:28 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:29.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:28 smithi083 ceph-mon[24755]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 2.9 MiB/s wr, 285 op/s 2024-03-11T20:40:29.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:28 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:29.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:28 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:29.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:28 smithi204 ceph-mon[27258]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 34 KiB/s rd, 2.9 MiB/s wr, 285 op/s 2024-03-11T20:40:30.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 133419 11.76 MB/sec execute 133 sec latency 41.565 ms 2024-03-11T20:40:30.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:40:30.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:40:30.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:40:30.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:40:30.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:40:30.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:29 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/1428269432' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:40:30.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:29 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:40:31.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 133988 11.79 MB/sec execute 134 sec latency 29.847 ms 2024-03-11T20:40:31.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:30 smithi083 ceph-mon[24755]: mgrmap e34: smithi083.jyxogi(active, since 10s), standbys: smithi204.qknpir 2024-03-11T20:40:31.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:30 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:31.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:30 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:31.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:30 smithi083 ceph-mon[24755]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.3 MiB/s wr, 221 op/s 2024-03-11T20:40:31.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:30 smithi204 ceph-mon[27258]: mgrmap e34: smithi083.jyxogi(active, since 10s), standbys: smithi204.qknpir 2024-03-11T20:40:31.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:30 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:31.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:30 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:31.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:30 smithi204 ceph-mon[27258]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.3 MiB/s wr, 221 op/s 2024-03-11T20:40:32.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 134295 11.78 MB/sec execute 135 sec latency 50.600 ms 2024-03-11T20:40:32.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:32 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:32.791 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:32 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:32.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:32 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:32.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:32 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:33.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 134618 11.83 MB/sec execute 136 sec latency 49.961 ms 2024-03-11T20:40:33.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:33 smithi083 ceph-mon[24755]: pgmap v9: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.2 MiB/s wr, 445 op/s 2024-03-11T20:40:33.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:33 smithi204 ceph-mon[27258]: pgmap v9: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.2 MiB/s wr, 445 op/s 2024-03-11T20:40:34.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 134873 11.79 MB/sec execute 137 sec latency 47.702 ms 2024-03-11T20:40:35.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135138 11.77 MB/sec execute 138 sec latency 76.281 ms 2024-03-11T20:40:35.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:35 smithi083 ceph-mon[24755]: pgmap v10: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.9 MiB/s wr, 406 op/s 2024-03-11T20:40:35.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:35 smithi204 ceph-mon[27258]: pgmap v10: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.9 MiB/s wr, 406 op/s 2024-03-11T20:40:36.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135423 11.71 MB/sec execute 139 sec latency 54.915 ms 2024-03-11T20:40:36.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:36 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:36.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:36 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:37.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135657 11.65 MB/sec execute 140 sec latency 89.690 ms 2024-03-11T20:40:37.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:37 smithi083 ceph-mon[24755]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-03-11T20:40:37.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:37 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:37.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:37 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:37.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:37 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:37.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:37 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:40:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:37 smithi204 ceph-mon[27258]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-03-11T20:40:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:37 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:37 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:37 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:37.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:37 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:40:38.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135980 11.61 MB/sec execute 141 sec latency 62.368 ms 2024-03-11T20:40:38.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:38 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:40:38.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:40:38.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:40:38.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:40:38.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:38.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:38 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:39.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 136335 11.59 MB/sec execute 142 sec latency 90.139 ms 2024-03-11T20:40:39.485 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:40:39.665 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:39 smithi083 ceph-mon[24755]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:40:39.665 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:39 smithi083 ceph-mon[24755]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:40:39.665 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:39 smithi083 ceph-mon[24755]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:40:39.665 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:39 smithi083 ceph-mon[24755]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:40:39.665 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:39 smithi083 ceph-mon[24755]: pgmap v12: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-03-11T20:40:39.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:39 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:40:39.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:39 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:40:39.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:39 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:40:39.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:39 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:40:39.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:39 smithi204 ceph-mon[27258]: pgmap v12: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-03-11T20:40:40.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 136714 11.56 MB/sec execute 143 sec latency 111.533 ms 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (105s) 10s ago 11m 15.2M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (10m) 10s ago 10m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (10m) 14s ago 10m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (82s) 10s ago 10m 45.6M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (4m) 10s ago 4m 37.7M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (4m) 14s ago 4m 16.0M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (40s) 10s ago 12m 478M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:40:40.650 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283 running (2m) 14s ago 10m 108M - 19.0.0-1979-g2b0101eb 4133f8bc41bb ce145ba7788d 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (12m) 10s ago 12m 55.3M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (10m) 14s ago 10m 45.9M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (4m) 10s ago 4m 262M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (4m) 14s ago 4m 85.5M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (79s) 10s ago 10m 8539k - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (50s) 14s ago 9m 7801k - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9m) 10s ago 9m 479M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 10s ago 8m 411M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 10s ago 8m 292M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 10s ago 7m 441M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (7m) 14s ago 7m 454M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (7m) 14s ago 7m 501M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:40:40.651 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (6m) 14s ago 6m 415M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:40:40.652 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (6m) 14s ago 6m 580M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:40:40.652 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (52s) 10s ago 9m 53.0M - 2.33.4 514e6a882f6e bb6d252d9ada 2024-03-11T20:40:40.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:40 smithi083 ceph-mon[24755]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-03-11T20:40:40.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:40 smithi083 ceph-mon[24755]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-03-11T20:40:40.695 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:40 smithi083 ceph-mon[24755]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:40.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:40 smithi204 ceph-mon[27258]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-03-11T20:40:40.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:40 smithi204 ceph-mon[27258]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-03-11T20:40:40.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:40 smithi204 ceph-mon[27258]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:41.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137046 11.55 MB/sec execute 144 sec latency 39.462 ms 2024-03-11T20:40:41.188 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:40:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:40:41.188 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:40:41.189 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:40:41.190 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:40:41.633 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "2/23 daemons upgraded", 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:40:41.634 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:40:41.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:41 smithi083 ceph-mon[24755]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:41.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:41 smithi083 ceph-mon[24755]: pgmap v13: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-03-11T20:40:41.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:41 smithi083 ceph-mon[24755]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:41.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:41 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/3056856061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:40:41.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:41 smithi204 ceph-mon[27258]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:41.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:41 smithi204 ceph-mon[27258]: pgmap v13: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-03-11T20:40:41.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:41 smithi204 ceph-mon[27258]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:41.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:41 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3056856061' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:40:42.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137414 11.53 MB/sec execute 145 sec latency 46.342 ms 2024-03-11T20:40:42.308 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:40:42.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:42 smithi083 ceph-mon[24755]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:42.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:42 smithi083 ceph-mon[24755]: from='client.? 172.21.15.83:0/979896972' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:40:42.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:42 smithi204 ceph-mon[27258]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:40:42.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:42 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/979896972' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:40:43.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137835 11.55 MB/sec execute 146 sec latency 34.429 ms 2024-03-11T20:40:43.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:43 smithi083 ceph-mon[24755]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.9 MiB/s wr, 629 op/s 2024-03-11T20:40:43.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:43 smithi204 ceph-mon[27258]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.9 MiB/s wr, 629 op/s 2024-03-11T20:40:44.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 138236 11.60 MB/sec execute 147 sec latency 31.463 ms 2024-03-11T20:40:44.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:44 smithi083 ceph-mon[24755]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-03-11T20:40:45.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 138579 11.59 MB/sec execute 148 sec latency 31.731 ms 2024-03-11T20:40:45.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:44 smithi204 ceph-mon[27258]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-03-11T20:40:46.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 139077 11.58 MB/sec execute 149 sec latency 31.605 ms 2024-03-11T20:40:47.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 139550 11.57 MB/sec execute 150 sec latency 79.577 ms 2024-03-11T20:40:47.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:47 smithi083 ceph-mon[24755]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-03-11T20:40:47.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:47 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:47.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:47 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:47.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:47 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:40:47.820 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:47 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:47.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:47 smithi204 ceph-mon[27258]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-03-11T20:40:47.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:47 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:47.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:47 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:47.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:47 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:40:47.902 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:47 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:48.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 140087 11.55 MB/sec execute 151 sec latency 86.004 ms 2024-03-11T20:40:48.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:48 smithi083 ceph-mon[24755]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:40:48.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:48 smithi204 ceph-mon[27258]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:40:49.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 140733 11.59 MB/sec execute 152 sec latency 43.475 ms 2024-03-11T20:40:49.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:49 smithi083 ceph-mon[24755]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-03-11T20:40:49.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:49 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:40:49.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:49 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:40:49.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:49 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:49.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:49 smithi204 ceph-mon[27258]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-03-11T20:40:49.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:49 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:40:49.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:49 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:40:49.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:49 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:40:50.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 141421 11.66 MB/sec execute 153 sec latency 25.459 ms 2024-03-11T20:40:50.858 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:50 smithi083 ceph-mon[24755]: Upgrade: Updating mgr.smithi204.qknpir 2024-03-11T20:40:50.858 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:50 smithi083 ceph-mon[24755]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:40:50.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:50 smithi204 ceph-mon[27258]: Upgrade: Updating mgr.smithi204.qknpir 2024-03-11T20:40:50.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:50 smithi204 ceph-mon[27258]: Deploying daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:40:51.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 142016 11.74 MB/sec execute 154 sec latency 48.741 ms 2024-03-11T20:40:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:51 smithi204 ceph-mon[27258]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-11T20:40:51.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:51 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:51.872 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:51 smithi083 ceph-mon[24755]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-03-11T20:40:51.872 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:51 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:40:52.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 142508 11.73 MB/sec execute 155 sec latency 23.889 ms 2024-03-11T20:40:53.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 143001 11.72 MB/sec execute 156 sec latency 26.470 ms 2024-03-11T20:40:53.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:53 smithi083 ceph-mon[24755]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-03-11T20:40:53.269 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:53 smithi204 ceph-mon[27258]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-03-11T20:40:54.020 INFO:teuthology.orchestra.run.smithi083.stdout: 5 143547 11.71 MB/sec execute 157 sec latency 48.792 ms 2024-03-11T20:40:55.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 143985 11.71 MB/sec execute 158 sec latency 40.346 ms 2024-03-11T20:40:55.409 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:55 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:55.410 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:55 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:55.410 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:55 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:55.410 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:55 smithi083 ceph-mon[24755]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 603 op/s 2024-03-11T20:40:55.619 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:55 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:55.619 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:55 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:40:55.619 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:55 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:40:55.620 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:55 smithi204 ceph-mon[27258]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 603 op/s 2024-03-11T20:40:56.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 144547 11.74 MB/sec execute 159 sec latency 43.075 ms 2024-03-11T20:40:57.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 144961 11.74 MB/sec execute 160 sec latency 61.344 ms 2024-03-11T20:40:57.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:57 smithi083 ceph-mon[24755]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.8 MiB/s wr, 708 op/s 2024-03-11T20:40:57.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:57 smithi204 ceph-mon[27258]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.8 MiB/s wr, 708 op/s 2024-03-11T20:40:58.021 INFO:teuthology.orchestra.run.smithi083.stdout: 5 145364 11.79 MB/sec execute 161 sec latency 30.072 ms 2024-03-11T20:40:59.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 145825 11.80 MB/sec execute 162 sec latency 42.013 ms 2024-03-11T20:40:59.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:40:59 smithi083 ceph-mon[24755]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 883 op/s 2024-03-11T20:40:59.789 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:40:59 smithi204 ceph-mon[27258]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 883 op/s 2024-03-11T20:41:00.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146125 11.75 MB/sec execute 163 sec latency 60.186 ms 2024-03-11T20:41:01.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146347 11.70 MB/sec execute 164 sec latency 84.408 ms 2024-03-11T20:41:01.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:01 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:01.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:01 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:01.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:01 smithi083 ceph-mon[24755]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 647 op/s 2024-03-11T20:41:01.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:01 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:01.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:01 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:01.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:01 smithi204 ceph-mon[27258]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 647 op/s 2024-03-11T20:41:02.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146602 11.68 MB/sec execute 165 sec latency 67.244 ms 2024-03-11T20:41:02.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:02 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:02.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:02 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:02.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:02 smithi083 ceph-mon[24755]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-03-11T20:41:02.772 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:02 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:02.772 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:02 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:02.772 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:02 smithi204 ceph-mon[27258]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-03-11T20:41:03.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146899 11.63 MB/sec execute 166 sec latency 52.276 ms 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:03.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:03 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2271009260' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:03.806 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:03 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:04.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 147252 11.59 MB/sec execute 167 sec latency 72.974 ms 2024-03-11T20:41:04.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:04 smithi083 ceph-mon[24755]: mgrmap e35: smithi083.jyxogi(active, since 44s), standbys: smithi204.qknpir 2024-03-11T20:41:04.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:04 smithi083 ceph-mon[24755]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-03-11T20:41:05.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 147835 11.64 MB/sec execute 168 sec latency 24.502 ms 2024-03-11T20:41:05.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:04 smithi204 ceph-mon[27258]: mgrmap e35: smithi083.jyxogi(active, since 44s), standbys: smithi204.qknpir 2024-03-11T20:41:05.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:04 smithi204 ceph-mon[27258]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-03-11T20:41:06.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 148484 11.68 MB/sec execute 169 sec latency 28.395 ms 2024-03-11T20:41:06.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.jyxogi"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.jyxogi"}]': finished 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi204.qknpir"}]: dispatch 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi204.qknpir"}]': finished 2024-03-11T20:41:06.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:06 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-11T20:41:06.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.jyxogi"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.jyxogi"}]': finished 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi204.qknpir"}]: dispatch 2024-03-11T20:41:06.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi204.qknpir"}]': finished 2024-03-11T20:41:06.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:06 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-11T20:41:07.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 148928 11.73 MB/sec execute 170 sec latency 42.359 ms 2024-03-11T20:41:07.812 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:07 smithi083 ceph-mon[24755]: Upgrade: Setting container_image for all mgr 2024-03-11T20:41:07.812 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:07 smithi083 ceph-mon[24755]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 565 op/s 2024-03-11T20:41:07.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:07 smithi204 ceph-mon[27258]: Upgrade: Setting container_image for all mgr 2024-03-11T20:41:07.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:07 smithi204 ceph-mon[27258]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 565 op/s 2024-03-11T20:41:08.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 149365 11.72 MB/sec execute 171 sec latency 25.064 ms 2024-03-11T20:41:09.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 149710 11.70 MB/sec execute 172 sec latency 51.559 ms 2024-03-11T20:41:09.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:09 smithi083 ceph-mon[24755]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-03-11T20:41:09.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:09 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:09.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:09 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:41:09.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:09 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:41:09.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:09 smithi083 ceph-mon[24755]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:09.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:09 smithi204 ceph-mon[27258]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-03-11T20:41:09.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:09 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:09.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:09 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:41:09.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:09 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:41:09.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:09 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:10.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150007 11.67 MB/sec execute 173 sec latency 53.797 ms 2024-03-11T20:41:10.842 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:10 smithi083 ceph-mon[24755]: Upgrade: Updating mon.smithi083 2024-03-11T20:41:10.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:10 smithi083 ceph-mon[24755]: Deploying daemon mon.smithi083 on smithi083 2024-03-11T20:41:10.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:10 smithi204 ceph-mon[27258]: Upgrade: Updating mon.smithi083 2024-03-11T20:41:10.927 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:10 smithi204 ceph-mon[27258]: Deploying daemon mon.smithi083 on smithi083 2024-03-11T20:41:11.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150426 11.65 MB/sec execute 174 sec latency 42.483 ms 2024-03-11T20:41:11.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:11 smithi083 ceph-mon[24755]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-03-11T20:41:11.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:11 smithi204 ceph-mon[27258]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-03-11T20:41:12.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150838 11.61 MB/sec execute 175 sec latency 59.126 ms 2024-03-11T20:41:12.371 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:12 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:41:12.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[24751]: 2024-03-11T20:41:12.624+0000 7fcdc55f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:41:12.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[24751]: 2024-03-11T20:41:12.624+0000 7fcdc55f3700 -1 mon.smithi083@0(leader) e2 *** Got Signal Terminated *** 2024-03-11T20:41:12.801 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:41:12.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:12 smithi083 podman[66072]: 2024-03-11 20:41:12.688974477 +0000 UTC m=+0.219165227 container died ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-type=git, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, distribution-scope=public, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-11T20:41:13.022 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151088 11.59 MB/sec execute 176 sec latency 74.804 ms 2024-03-11T20:41:13.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:13 smithi083 podman[66072]: 2024-03-11 20:41:13.377712079 +0000 UTC m=+0.907902840 container cleanup ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, architecture=x86_64, distribution-scope=public, release=754, ceph=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc.) 2024-03-11T20:41:13.396 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:13 smithi083 bash[66072]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (2m) 10s ago 11m 15.2M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (11m) 10s ago 11m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (10m) 13s ago 10m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (116s) 10s ago 11m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (5m) 10s ago 5m 38.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:41:13.738 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (5m) 13s ago 5m 16.2M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (73s) 10s ago 12m 494M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (20s) 13s ago 10m 60.5M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (12m) 10s ago 12m 56.0M 2048M 17.2.0 e1d6a67b021e ba49e4ddedb2 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (10m) 13s ago 10m 46.5M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (5m) 10s ago 5m 264M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (5m) 13s ago 5m 87.1M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (112s) 10s ago 11m 11.7M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (83s) 13s ago 10m 11.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10m) 10s ago 10m 507M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9m) 10s ago 9m 441M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 10s ago 8m 326M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8m) 10s ago 8m 468M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (8m) 13s ago 8m 480M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:41:13.739 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (7m) 13s ago 7m 562M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:41:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (7m) 13s ago 7m 468M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:41:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (6m) 13s ago 6m 365M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:41:13.740 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (27s) 10s ago 10m 51.0M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:41:13.901 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:13 smithi083 podman[66105]: 2024-03-11 20:41:13.697351864 +0000 UTC m=+0.998810236 container remove ba49e4ddedb2964a428c0bd88acd17ef522e4b76f3d58f78384a6857ab52dc45 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, GIT_BRANCH=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8, ceph=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-03-11T20:41:14.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151488 11.61 MB/sec execute 177 sec latency 50.706 ms 2024-03-11T20:41:14.155 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service: Deactivated successfully. 2024-03-11T20:41:14.155 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 systemd[1]: Stopped Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:41:14.155 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service: Consumed 16.545s CPU time. 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:41:14.375 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:41:14.376 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:41:14.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 systemd[1]: Starting Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "2/23 daemons upgraded", 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mon daemons", 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:41:14.852 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:41:14.935 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 podman[66324]: 2024-03-11 20:41:14.635606075 +0000 UTC m=+0.028635304 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:41:14.935 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:14 smithi083 podman[66324]: 2024-03-11 20:41:14.81019679 +0000 UTC m=+0.203226003 container create 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-11T20:41:15.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151891 11.60 MB/sec execute 178 sec latency 33.527 ms 2024-03-11T20:41:15.428 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:41:15.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 podman[66324]: 2024-03-11 20:41:15.323625682 +0000 UTC m=+0.716654914 container init 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e) 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 podman[66324]: 2024-03-11 20:41:15.335535088 +0000 UTC m=+0.728564310 container start 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307) 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: set uid:gid to 167:167 (ceph:ceph) 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev), process ceph-mon, pid 2 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: pidfile_write: ignore empty --pid-file 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: load: jerasure load: lrc 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: RocksDB version: 7.9.2 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Git sha 0 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Compile date 2024-03-10 23:25:48 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: DB SUMMARY 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: DB Session ID: 6HY6RSN57TWDFM1EDV92 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: CURRENT file: CURRENT 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: IDENTITY file: IDENTITY 2024-03-11T20:41:15.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi083/store.db dir, Total Num: 1, files: 000030.sst 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi083/store.db: 000028.log size: 3148002 ; 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.error_if_exists: 0 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.create_if_missing: 0 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.paranoid_checks: 1 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.env: 0x55dbdb7d6b20 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.fs: PosixFileSystem 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.info_log: 0x55dbdd0bb780 2024-03-11T20:41:15.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_file_opening_threads: 16 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.statistics: (nil) 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.use_fsync: 0 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_log_file_size: 0 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.keep_log_file_num: 1000 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.recycle_log_file_num: 0 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_fallocate: 1 2024-03-11T20:41:15.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_mmap_reads: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_mmap_writes: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.use_direct_reads: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.create_missing_column_families: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.db_log_dir: 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.wal_dir: 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-11T20:41:15.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.advise_random_on_open: 1 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.db_write_buffer_size: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.write_buffer_manager: 0x55dbdd08f400 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.rate_limiter: (nil) 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.wal_recovery_mode: 2 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enable_thread_tracking: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enable_pipelined_write: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.unordered_write: 0 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-11T20:41:15.452 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.row_cache: None 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.wal_filter: None 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_ingest_behind: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.two_write_queues: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.manual_wal_flush: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.wal_compression: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.atomic_flush: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.log_readahead_size: 0 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-11T20:41:15.453 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.best_efforts_recovery: 0 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.allow_data_in_errors: 0 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.db_host_id: __hostname__ 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_background_jobs: 2 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_background_compactions: -1 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_subcompactions: 1 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_total_wal_size: 0 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-11T20:41:15.454 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_open_files: -1 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bytes_per_sync: 0 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_readahead_size: 0 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_background_flushes: -1 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Compression algorithms supported: 2024-03-11T20:41:15.455 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kZSTD supported: 0 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kXpressCompression supported: 0 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kLZ4HCCompression supported: 1 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kZlibCompression supported: 1 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kSnappyCompression supported: 1 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kLZ4Compression supported: 1 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: kBZip2Compression supported: 0 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000015 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.merge_operator: 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_filter: None 2024-03-11T20:41:15.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_filter_factory: None 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.sst_partitioner_factory: None 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dbdd0bb8a0) 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_type: 0 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_index_type: 0 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_shortening: 1 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: checksum: 4 2024-03-11T20:41:15.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: no_block_cache: 0 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache: 0x55dbdd0c1090 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_options: 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: capacity : 536870912 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_shard_bits : 4 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: strict_capacity_limit : 0 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_compressed: (nil) 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: persistent_cache: (nil) 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size: 4096 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size_deviation: 10 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_restart_interval: 16 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_block_restart_interval: 1 2024-03-11T20:41:15.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_block_size: 4096 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: partition_filters: 0 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: use_delta_encoding: 1 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: filter_policy: bloomfilter 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: whole_key_filtering: 1 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: verify_compression: 0 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: format_version: 5 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_index_compression: 1 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_align: 0 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_auto_readahead_size: 262144 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: prepopulate_block_cache: 0 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: initial_auto_readahead_size: 8192 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_file_reads_for_auto_readahead: 2 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.write_buffer_size: 33554432 2024-03-11T20:41:15.459 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_write_buffer_number: 2 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression: NoCompression 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression: Disabled 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.prefix_extractor: nullptr 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.num_levels: 7 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-11T20:41:15.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.level: 32767 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.strategy: 0 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.enabled: false 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-11T20:41:15.461 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.target_file_size_base: 67108864 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-11T20:41:15.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.arena_block_size: 1048576 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.disable_auto_compactions: 0 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-11T20:41:15.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.table_properties_collectors: 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.inplace_update_support: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.bloom_locality: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.max_successive_merges: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.paranoid_file_checks: 0 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.force_consistency_checks: 1 2024-03-11T20:41:15.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.report_bg_io_stats: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.ttl: 2592000 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enable_blob_files: false 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.min_blob_size: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_file_size: 268435456 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.blob_file_starting_level: 0 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-11T20:41:15.465 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11820, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 208dcb78-0289-4857-9f4e-7d78f2636466 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189675378962, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189675389759, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2900908, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11814, "largest_seqno": 13064, "table_properties": {"data_size": 2894808, "index_size": 3737, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 11947, "raw_average_key_size": 22, "raw_value_size": 2883357, "raw_average_value_size": 5502, "num_data_blocks": 170, "num_entries": 524, "num_filter_entries": 524, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1710189675, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "208dcb78-0289-4857-9f4e-7d78f2636466", "db_session_id": "6HY6RSN57TWDFM1EDV92", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189675390048, "job": 1, "event": "recovery_finished"} 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi083/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dbdd1b0000 2024-03-11T20:41:15.466 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: DB pointer 0x55dbdd19e000 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** DB Stats ** 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-11T20:41:15.467 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 1/0 2.77 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L6 1/0 6.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 9.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 516.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-03-11T20:41:15.468 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 169.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 169.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Block cache BinnedLRUCache@0x55dbdd0c1090#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: starting mon.smithi083 rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi083 fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???) e2 preinit fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).mds e7 new map 2024-03-11T20:41:15.469 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).mds e7 print_map 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: e7 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: legacy client fscid: 1 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Filesystem 'foofs' (1) 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: fs_name foofs 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: epoch 5 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: created 2024-03-11T20:35:41.637680+0000 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: modified 2024-03-11T20:35:47.660817+0000 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: tableserver 0 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: root 0 2024-03-11T20:41:15.470 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: session_timeout 60 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: session_autoclose 300 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_file_size 1099511627776 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_xattr_size 65536 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: required_client_features {} 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: last_failure 0 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: last_failure_osd_epoch 0 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_mds 1 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: in 0 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: up {0=14498} 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: failed 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: damaged 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: stopped 2024-03-11T20:41:15.471 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_pools [3] 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_pool 2 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: inline_data disabled 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: balancer 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: bal_rank_mask -1 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: standby_count_wanted 1 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: [mds.foofs.smithi083.motgne{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] compat {c=[1],r=[1],i=[7ff]}] 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Standby daemons: 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: [mds.foofs.smithi204.evhral{-1:24293} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] compat {c=[1],r=[1],i=[7ff]}] 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:15.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 ceph-mon[66365]: mon.smithi083@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-03-11T20:41:15.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 bash[66324]: 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c 2024-03-11T20:41:15.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:15 smithi083 systemd[1]: Started Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:41:16.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 152484 11.67 MB/sec execute 179 sec latency 42.893 ms 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.14860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.24597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.24601 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3704209139' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 505 op/s 2024-03-11T20:41:16.201 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.24609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1705476401' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: mon.smithi083 calling monitor election 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: osdmap e72: 8 total, 8 up, 8 in 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: mgrmap e35: smithi083.jyxogi(active, since 56s), standbys: smithi204.qknpir 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: overall HEALTH_OK 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: from='mgr.24521 ' entity='' 2024-03-11T20:41:16.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:16 smithi083 ceph-mon[66365]: mgrmap e36: smithi083.jyxogi(active, since 56s), standbys: smithi204.qknpir 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='mgr.24521 172.21.15.83:0/1825518497' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.14860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.24597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.24601 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/3704209139' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 505 op/s 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.24609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='client.? 172.21.15.83:0/1705476401' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: mon.smithi083 calling monitor election 2024-03-11T20:41:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: osdmap e72: 8 total, 8 up, 8 in 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: mgrmap e35: smithi083.jyxogi(active, since 56s), standbys: smithi204.qknpir 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: overall HEALTH_OK 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: from='mgr.24521 ' entity='' 2024-03-11T20:41:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:16 smithi204 ceph-mon[27258]: mgrmap e36: smithi083.jyxogi(active, since 56s), standbys: smithi204.qknpir 2024-03-11T20:41:17.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 153097 11.69 MB/sec execute 180 sec latency 42.509 ms 2024-03-11T20:41:18.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 153455 11.68 MB/sec execute 181 sec latency 39.645 ms 2024-03-11T20:41:18.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:18 smithi204 ceph-mon[27258]: mgrmap e37: smithi083.jyxogi(active, since 57s), standbys: smithi204.qknpir 2024-03-11T20:41:18.217 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:18 smithi083 ceph-mon[66365]: mgrmap e37: smithi083.jyxogi(active, since 57s), standbys: smithi204.qknpir 2024-03-11T20:41:19.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 153872 11.66 MB/sec execute 182 sec latency 57.310 ms 2024-03-11T20:41:20.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 154311 11.62 MB/sec execute 183 sec latency 47.884 ms 2024-03-11T20:41:21.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 154629 11.60 MB/sec execute 184 sec latency 39.208 ms 2024-03-11T20:41:22.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 155131 11.63 MB/sec execute 185 sec latency 41.093 ms 2024-03-11T20:41:23.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 155628 11.65 MB/sec execute 186 sec latency 25.276 ms 2024-03-11T20:41:24.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156302 11.73 MB/sec execute 187 sec latency 22.997 ms 2024-03-11T20:41:24.280 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:41:24.280 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:41:24.280 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:41:24.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:41:24.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:41:24.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:24 smithi083 ceph-mon[66365]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:41:24.610 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir restarted 2024-03-11T20:41:24.610 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: Standby manager daemon smithi204.qknpir started 2024-03-11T20:41:24.610 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/crt"}]: dispatch 2024-03-11T20:41:24.610 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-11T20:41:24.611 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi204.qknpir/key"}]: dispatch 2024-03-11T20:41:24.611 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:24 smithi204 ceph-mon[27258]: from='mgr.? 172.21.15.204:0/2005305634' entity='mgr.smithi204.qknpir' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-11T20:41:25.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156690 11.71 MB/sec execute 188 sec latency 37.495 ms 2024-03-11T20:41:25.286 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:25 smithi083 ceph-mon[66365]: mgrmap e38: smithi083.jyxogi(active, since 64s), standbys: smithi204.qknpir 2024-03-11T20:41:25.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:25 smithi204 ceph-mon[27258]: mgrmap e38: smithi083.jyxogi(active, since 64s), standbys: smithi204.qknpir 2024-03-11T20:41:26.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156946 11.67 MB/sec execute 189 sec latency 55.421 ms 2024-03-11T20:41:26.920 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: osdmap e73: 8 total, 8 up, 8 in 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: mgrmap e39: smithi083.jyxogi(active, starting, since 0.0112161s), standbys: smithi204.qknpir 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:41:26.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:41:26.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:26 smithi083 ceph-mon[66365]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:41:27.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157210 11.65 MB/sec execute 190 sec latency 63.340 ms 2024-03-11T20:41:27.135 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: Active manager daemon smithi083.jyxogi restarted 2024-03-11T20:41:27.135 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: Activating manager daemon smithi083.jyxogi 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: osdmap e73: 8 total, 8 up, 8 in 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: mgrmap e39: smithi083.jyxogi(active, starting, since 0.0112161s), standbys: smithi204.qknpir 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi083.jyxogi", "id": "smithi083.jyxogi"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr metadata", "who": "smithi204.qknpir", "id": "smithi204.qknpir"}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:41:27.136 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-11T20:41:27.137 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:26 smithi204 ceph-mon[27258]: Manager daemon smithi083.jyxogi is now available 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:41:27.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:41:27.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:41:27.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:27 smithi083 ceph-mon[66365]: mgrmap e40: smithi083.jyxogi(active, since 1.01571s), standbys: smithi204.qknpir 2024-03-11T20:41:28.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157650 11.62 MB/sec execute 191 sec latency 98.707 ms 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/mirror_snapshot_schedule"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.jyxogi/trash_purge_schedule"}]: dispatch 2024-03-11T20:41:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:27 smithi204 ceph-mon[27258]: mgrmap e40: smithi083.jyxogi(active, since 1.01571s), standbys: smithi204.qknpir 2024-03-11T20:41:28.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:28 smithi083 ceph-mon[66365]: [11/Mar/2024:20:41:27] ENGINE Bus STARTING 2024-03-11T20:41:28.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:28 smithi083 ceph-mon[66365]: [11/Mar/2024:20:41:27] ENGINE Serving on http://172.21.15.83:8765 2024-03-11T20:41:28.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:28 smithi083 ceph-mon[66365]: pgmap v3: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:28.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:28 smithi083 ceph-mon[66365]: [11/Mar/2024:20:41:27] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:41:28.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:28 smithi083 ceph-mon[66365]: [11/Mar/2024:20:41:27] ENGINE Bus STARTED 2024-03-11T20:41:29.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 158119 11.61 MB/sec execute 192 sec latency 40.286 ms 2024-03-11T20:41:29.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:28 smithi204 ceph-mon[27258]: [11/Mar/2024:20:41:27] ENGINE Bus STARTING 2024-03-11T20:41:29.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:28 smithi204 ceph-mon[27258]: [11/Mar/2024:20:41:27] ENGINE Serving on http://172.21.15.83:8765 2024-03-11T20:41:29.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:28 smithi204 ceph-mon[27258]: pgmap v3: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:29.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:28 smithi204 ceph-mon[27258]: [11/Mar/2024:20:41:27] ENGINE Serving on https://172.21.15.83:7150 2024-03-11T20:41:29.153 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:28 smithi204 ceph-mon[27258]: [11/Mar/2024:20:41:27] ENGINE Bus STARTED 2024-03-11T20:41:29.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:29 smithi083 ceph-mon[66365]: pgmap v4: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:29.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:29 smithi083 ceph-mon[66365]: mgrmap e41: smithi083.jyxogi(active, since 3s), standbys: smithi204.qknpir 2024-03-11T20:41:30.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 158771 11.66 MB/sec execute 193 sec latency 22.857 ms 2024-03-11T20:41:30.103 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:29 smithi204 ceph-mon[27258]: pgmap v4: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:30.103 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:29 smithi204 ceph-mon[27258]: mgrmap e41: smithi083.jyxogi(active, since 3s), standbys: smithi204.qknpir 2024-03-11T20:41:31.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 159285 11.70 MB/sec execute 194 sec latency 42.360 ms 2024-03-11T20:41:31.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:31 smithi083 ceph-mon[66365]: pgmap v5: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:31.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:31 smithi083 ceph-mon[66365]: mgrmap e42: smithi083.jyxogi(active, since 4s), standbys: smithi204.qknpir 2024-03-11T20:41:31.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:31 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:31.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:31 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:32.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 159710 11.71 MB/sec execute 195 sec latency 40.284 ms 2024-03-11T20:41:32.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:31 smithi204 ceph-mon[27258]: pgmap v5: 97 pgs: 97 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-11T20:41:32.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:31 smithi204 ceph-mon[27258]: mgrmap e42: smithi083.jyxogi(active, since 4s), standbys: smithi204.qknpir 2024-03-11T20:41:32.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:31 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:32.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:31 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:33.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 160128 11.71 MB/sec execute 196 sec latency 46.996 ms 2024-03-11T20:41:33.999 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:33 smithi083 ceph-mon[66365]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 49 KiB/s rd, 8.0 MiB/s wr, 723 op/s 2024-03-11T20:41:33.999 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:33 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:33.999 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:33 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:34.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 160436 11.68 MB/sec execute 197 sec latency 40.049 ms 2024-03-11T20:41:34.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:33 smithi204 ceph-mon[27258]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 49 KiB/s rd, 8.0 MiB/s wr, 723 op/s 2024-03-11T20:41:34.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:33 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:34.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:33 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:35.024 INFO:teuthology.orchestra.run.smithi083.stdout: 5 160818 11.66 MB/sec execute 198 sec latency 41.904 ms 2024-03-11T20:41:36.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:35 smithi083 ceph-mon[66365]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 5.7 MiB/s wr, 517 op/s 2024-03-11T20:41:36.023 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161166 11.63 MB/sec execute 199 sec latency 55.094 ms 2024-03-11T20:41:36.068 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:35 smithi204 ceph-mon[27258]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 5.7 MiB/s wr, 517 op/s 2024-03-11T20:41:37.024 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161522 11.61 MB/sec execute 200 sec latency 76.137 ms 2024-03-11T20:41:37.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:37 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:37.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:37 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:37.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:37 smithi083 ceph-mon[66365]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 4.4 MiB/s wr, 402 op/s 2024-03-11T20:41:38.024 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161822 11.59 MB/sec execute 201 sec latency 35.344 ms 2024-03-11T20:41:38.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:37 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:38.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:37 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:38.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:37 smithi204 ceph-mon[27258]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 4.4 MiB/s wr, 402 op/s 2024-03-11T20:41:39.024 INFO:teuthology.orchestra.run.smithi083.stdout: 5 162309 11.61 MB/sec execute 202 sec latency 31.508 ms 2024-03-11T20:41:39.355 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:39 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:39.355 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:39 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:39.355 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:39 smithi083 ceph-mon[66365]: pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 582 op/s 2024-03-11T20:41:39.677 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:39 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:39.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:39 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:39.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:39 smithi204 ceph-mon[27258]: pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.6 MiB/s wr, 582 op/s 2024-03-11T20:41:40.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 162706 11.61 MB/sec execute 203 sec latency 50.477 ms 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:41:40.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:40 smithi083 ceph-mon[66365]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.conf 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.conf 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:41:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:40 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.conf 2024-03-11T20:41:41.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163004 11.63 MB/sec execute 204 sec latency 59.472 ms 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: pgmap v10: 97 pgs: 97 active+clean; 325 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.2 MiB/s wr, 534 op/s 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:41 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: Updating smithi204:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: Updating smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: Updating smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/config/ceph.client.admin.keyring 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:41.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: pgmap v10: 97 pgs: 97 active+clean; 325 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.2 MiB/s wr, 534 op/s 2024-03-11T20:41:41.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:41.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:41:41.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-11T20:41:42.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163370 11.63 MB/sec execute 205 sec latency 46.344 ms 2024-03-11T20:41:42.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:42 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:42.952 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:43.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163924 11.63 MB/sec execute 206 sec latency 18.958 ms 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: Upgrade: Updating mon.smithi204 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:43.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:43 smithi083 ceph-mon[66365]: Deploying daemon mon.smithi204 on smithi204 2024-03-11T20:41:44.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 164646 11.64 MB/sec execute 207 sec latency 25.145 ms 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: Upgrade: Updating mon.smithi204 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: from='mgr.24621 ' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:41:44.168 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:43 smithi204 ceph-mon[27258]: Deploying daemon mon.smithi204 on smithi204 2024-03-11T20:41:45.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 165392 11.66 MB/sec execute 208 sec latency 54.394 ms 2024-03-11T20:41:45.647 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:45 smithi204 systemd[1]: Stopping Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:41:45.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204[27236]: 2024-03-11T20:41:45.651+0000 7f9ae8447700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi204 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:41:45.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204[27236]: 2024-03-11T20:41:45.651+0000 7f9ae8447700 -1 mon.smithi204@1(peon) e2 *** Got Signal Terminated *** 2024-03-11T20:41:45.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:45 smithi204 podman[53115]: 2024-03-11 20:41:45.714219265 +0000 UTC m=+0.184533929 container died ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-11T20:41:45.960 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:41:46.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166017 11.70 MB/sec execute 209 sec latency 52.991 ms 2024-03-11T20:41:46.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 podman[53115]: 2024-03-11 20:41:46.32906275 +0000 UTC m=+0.799377414 container cleanup ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_BRANCH=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream) 2024-03-11T20:41:46.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 bash[53115]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204 2024-03-11T20:41:46.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 podman[53129]: 2024-03-11 20:41:46.580973138 +0000 UTC m=+0.865239328 container remove ccfbbf8a7768cb7e546f9fa58aed2e6085e4d715429a8267d908f730aa7c5127 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-11T20:41:46.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi204.service: Deactivated successfully. 2024-03-11T20:41:46.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 systemd[1]: Stopped Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:41:46.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi204.service: Consumed 7.382s CPU time. 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (2m) 8s ago 12m 17.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (12m) 8s ago 12m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (11m) 15s ago 11m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (2m) 8s ago 11m 45.8M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (6m) 8s ago 6m 39.3M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:41:46.958 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (5m) 15s ago 5m 16.4M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (106s) 8s ago 13m 479M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (53s) 15s ago 11m 447M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (32s) 8s ago 13m 31.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (11m) 15s ago 11m 53.9M 2048M 17.2.0 e1d6a67b021e ccfbbf8a7768 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (5m) 8s ago 5m 276M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (5m) 15s ago 5m 87.1M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (2m) 8s ago 11m 9009k - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (116s) 15s ago 11m 15.4M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10m) 8s ago 10m 574M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10m) 8s ago 10m 489M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9m) 8s ago 9m 353M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9m) 8s ago 9m 504M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (8m) 15s ago 8m 508M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:41:46.959 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (8m) 15s ago 8m 330M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:41:46.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (7m) 15s ago 7m 511M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:41:46.960 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (7m) 15s ago 7m 412M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:41:46.960 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (60s) 8s ago 10m 57.8M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:41:47.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166559 11.75 MB/sec execute 210 sec latency 38.566 ms 2024-03-11T20:41:47.248 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:46 smithi204 systemd[1]: Starting Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:41:47.529 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 podman[53253]: 2024-03-11 20:41:47.158117883 +0000 UTC m=+0.023143165 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:41:47.529 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 podman[53253]: 2024-03-11 20:41:47.282718964 +0000 UTC m=+0.147744217 container create 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=main-2b0101e, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:41:47.529 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 podman[53253]: 2024-03-11 20:41:47.536185061 +0000 UTC m=+0.401210327 container init 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True) 2024-03-11T20:41:47.545 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:41:47.546 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:41:47.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-03-11T20:41:47.547 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 3 2024-03-11T20:41:47.547 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:41:47.547 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:41:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 podman[53253]: 2024-03-11 20:41:47.540094412 +0000 UTC m=+0.405119677 container start 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:41:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: set uid:gid to 167:167 (ceph:ceph) 2024-03-11T20:41:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev), process ceph-mon, pid 2 2024-03-11T20:41:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: pidfile_write: ignore empty --pid-file 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: load: jerasure load: lrc 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: RocksDB version: 7.9.2 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Git sha 0 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Compile date 2024-03-10 23:25:48 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: DB SUMMARY 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: DB Session ID: WLT4OR0RWP73KCUCEPS8 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: CURRENT file: CURRENT 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: IDENTITY file: IDENTITY 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi204/store.db dir, Total Num: 1, files: 000024.sst 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi204/store.db: 000022.log size: 7958294 ; 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.error_if_exists: 0 2024-03-11T20:41:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.create_if_missing: 0 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.paranoid_checks: 1 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.env: 0x55de95f5eb20 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.fs: PosixFileSystem 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.info_log: 0x55de98227780 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_file_opening_threads: 16 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.statistics: (nil) 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.use_fsync: 0 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_log_file_size: 0 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-11T20:41:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.keep_log_file_num: 1000 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.recycle_log_file_num: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_fallocate: 1 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_mmap_reads: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_mmap_writes: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.use_direct_reads: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.create_missing_column_families: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.db_log_dir: 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.wal_dir: 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-11T20:41:47.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.advise_random_on_open: 1 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.db_write_buffer_size: 0 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.write_buffer_manager: 0x55de981fb400 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.rate_limiter: (nil) 2024-03-11T20:41:47.932 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.wal_recovery_mode: 2 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enable_thread_tracking: 0 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enable_pipelined_write: 0 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.unordered_write: 0 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-11T20:41:47.933 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.row_cache: None 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.wal_filter: None 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_ingest_behind: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.two_write_queues: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.manual_wal_flush: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.wal_compression: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.atomic_flush: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-11T20:41:47.934 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.log_readahead_size: 0 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.best_efforts_recovery: 0 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.allow_data_in_errors: 0 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.db_host_id: __hostname__ 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_background_jobs: 2 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_background_compactions: -1 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_subcompactions: 1 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-11T20:41:47.936 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_total_wal_size: 0 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_open_files: -1 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bytes_per_sync: 0 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_readahead_size: 0 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_background_flushes: -1 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Compression algorithms supported: 2024-03-11T20:41:47.937 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kZSTD supported: 0 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kXpressCompression supported: 0 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kLZ4HCCompression supported: 1 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kZlibCompression supported: 1 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kSnappyCompression supported: 1 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kLZ4Compression supported: 1 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: kBZip2Compression supported: 0 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi204/store.db/MANIFEST-000009 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.merge_operator: 2024-03-11T20:41:47.938 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_filter: None 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_filter_factory: None 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.sst_partitioner_factory: None 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55de982278a0) 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: cache_index_and_filter_blocks: 1 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: pin_top_level_index_and_filter: 1 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_type: 0 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: data_block_index_type: 0 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_shortening: 1 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-11T20:41:47.939 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: checksum: 4 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: no_block_cache: 0 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache: 0x55de9822d090 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_name: BinnedLRUCache 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_options: 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: capacity : 536870912 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: num_shard_bits : 4 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: strict_capacity_limit : 0 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: high_pri_pool_ratio: 0.000 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_cache_compressed: (nil) 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: persistent_cache: (nil) 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_size: 4096 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_size_deviation: 10 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_restart_interval: 16 2024-03-11T20:41:47.940 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: index_block_restart_interval: 1 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: metadata_block_size: 4096 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: partition_filters: 0 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: use_delta_encoding: 1 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: filter_policy: bloomfilter 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: whole_key_filtering: 1 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: verify_compression: 0 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: read_amp_bytes_per_bit: 0 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: format_version: 5 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: enable_index_compression: 1 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: block_align: 0 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: max_auto_readahead_size: 262144 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: prepopulate_block_cache: 0 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: initial_auto_readahead_size: 8192 2024-03-11T20:41:47.941 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: num_file_reads_for_auto_readahead: 2 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.write_buffer_size: 33554432 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_write_buffer_number: 2 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression: NoCompression 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression: Disabled 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.prefix_extractor: nullptr 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.num_levels: 7 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-11T20:41:47.942 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-11T20:41:47.943 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.level: 32767 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.strategy: 0 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.enabled: false 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.target_file_size_base: 67108864 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-11T20:41:47.944 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-11T20:41:47.945 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.arena_block_size: 1048576 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.disable_auto_compactions: 0 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-11T20:41:47.946 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.table_properties_collectors: 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.inplace_update_support: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.bloom_locality: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.max_successive_merges: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.paranoid_file_checks: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.force_consistency_checks: 1 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.report_bg_io_stats: 0 2024-03-11T20:41:47.947 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.ttl: 2592000 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enable_blob_files: false 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.min_blob_size: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_file_size: 268435456 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.blob_file_starting_level: 0 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-11T20:41:47.948 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi204/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12195, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bb54a397-7188-4558-bd37-3e87191547f2 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189707592298, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189707613219, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4850778, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12192, "largest_seqno": 14133, "table_properties": {"data_size": 4842365, "index_size": 5473, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1925, "raw_key_size": 17024, "raw_average_key_size": 22, "raw_value_size": 4826137, "raw_average_value_size": 6469, "num_data_blocks": 249, "num_entries": 746, "num_filter_entries": 746, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1710189707, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bb54a397-7188-4558-bd37-3e87191547f2", "db_session_id": "WLT4OR0RWP73KCUCEPS8", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1710189707613364, "job": 1, "event": "recovery_finished"} 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi204/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55de9831e000 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: DB pointer 0x55de9830a000 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-11T20:41:47.949 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** DB Stats ** 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: L0 1/0 4.63 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: L6 1/0 6.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-11T20:41:47.950 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Sum 2/0 11.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 516.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** Compaction Stats [default] ** 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 516.5 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-11T20:41:47.951 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Cumulative compaction: 0.00 GB write, 134.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Interval compaction: 0.00 GB write, 134.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Block cache BinnedLRUCache@0x55de9822d090#2 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: starting mon.smithi204 rank 1 at public addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] at bind addrs [v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi204 fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???) e2 preinit fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).mds e7 new map 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).mds e7 print_map 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: e7 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-11T20:41:47.952 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: legacy client fscid: 1 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Filesystem 'foofs' (1) 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: fs_name foofs 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: epoch 5 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: created 2024-03-11T20:35:41.637680+0000 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: modified 2024-03-11T20:35:47.660817+0000 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: tableserver 0 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: root 0 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: session_timeout 60 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: session_autoclose 300 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: max_file_size 1099511627776 2024-03-11T20:41:47.953 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: max_xattr_size 65536 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: required_client_features {} 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: last_failure 0 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: last_failure_osd_epoch 0 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: max_mds 1 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: in 0 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: up {0=14498} 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: failed 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: damaged 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: stopped 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: data_pools [3] 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: metadata_pool 2 2024-03-11T20:41:47.954 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: inline_data disabled 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: balancer 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: bal_rank_mask -1 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: standby_count_wanted 1 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: [mds.foofs.smithi083.motgne{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.83:6834/1821858523,v1:172.21.15.83:6835/1821858523] compat {c=[1],r=[1],i=[7ff]}] 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: Standby daemons: 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout: [mds.foofs.smithi204.evhral{-1:24293} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] compat {c=[1],r=[1],i=[7ff]}] 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-03-11T20:41:47.955 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 ceph-mon[53271]: mon.smithi204@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-03-11T20:41:47.956 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 bash[53253]: 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 2024-03-11T20:41:47.956 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:47 smithi204 systemd[1]: Started Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:41:48.002 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "3/23 daemons upgraded", 2024-03-11T20:41:48.003 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mon daemons", 2024-03-11T20:41:48.003 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:41:48.003 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:41:48.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166911 11.75 MB/sec execute 211 sec latency 46.054 ms 2024-03-11T20:41:48.583 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:41:49.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:48 smithi083 ceph-mon[66365]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-03-11T20:41:49.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:48 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:49.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:48 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:49.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167211 11.73 MB/sec execute 212 sec latency 52.082 ms 2024-03-11T20:41:49.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:48 smithi204 ceph-mon[53271]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-03-11T20:41:49.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:48 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:49.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:48 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:50.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167409 11.68 MB/sec execute 213 sec latency 53.703 ms 2024-03-11T20:41:50.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 834 op/s 2024-03-11T20:41:50.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:50.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:50.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: mon.smithi083 calling monitor election 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: mon.smithi204 calling monitor election 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: monmap e3: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: osdmap e73: 8 total, 8 up, 8 in 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: mgrmap e42: smithi083.jyxogi(active, since 22s), standbys: smithi204.qknpir 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: overall HEALTH_OK 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:50.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:49 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:50.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 834 op/s 2024-03-11T20:41:50.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-03-11T20:41:50.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mon metadata", "id": "smithi204"}]: dispatch 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: mon.smithi083 calling monitor election 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: mon.smithi204 calling monitor election 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: mon.smithi083 is new leader, mons smithi083,smithi204 in quorum (ranks 0,1) 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: monmap e3: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi204=[v2:172.21.15.204:3300/0,v1:172.21.15.204:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: osdmap e73: 8 total, 8 up, 8 in 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: mgrmap e42: smithi083.jyxogi(active, since 22s), standbys: smithi204.qknpir 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: overall HEALTH_OK 2024-03-11T20:41:50.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:50.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:50.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:49 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:41:51.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167657 11.65 MB/sec execute 214 sec latency 65.811 ms 2024-03-11T20:41:51.970 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:51 smithi083 ceph-mon[66365]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.5 MiB/s wr, 601 op/s 2024-03-11T20:41:52.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168009 11.65 MB/sec execute 215 sec latency 47.212 ms 2024-03-11T20:41:52.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:51 smithi204 ceph-mon[53271]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.5 MiB/s wr, 601 op/s 2024-03-11T20:41:53.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168373 11.61 MB/sec execute 216 sec latency 90.828 ms 2024-03-11T20:41:53.725 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:53 smithi204 ceph-mon[53271]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 794 op/s 2024-03-11T20:41:53.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:53 smithi083 ceph-mon[66365]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 794 op/s 2024-03-11T20:41:54.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168717 11.59 MB/sec execute 217 sec latency 48.001 ms 2024-03-11T20:41:55.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168955 11.56 MB/sec execute 218 sec latency 52.242 ms 2024-03-11T20:41:55.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:55.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:55.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:55.410 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:56.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169244 11.56 MB/sec execute 219 sec latency 87.791 ms 2024-03-11T20:41:56.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:56 smithi083 ceph-mon[66365]: pgmap v17: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s 2024-03-11T20:41:56.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:56.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:56.314 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:56 smithi204 ceph-mon[53271]: pgmap v17: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s 2024-03-11T20:41:56.314 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:56.315 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:41:57.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169504 11.54 MB/sec execute 220 sec latency 112.998 ms 2024-03-11T20:41:57.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:57.290 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:41:58.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169831 11.55 MB/sec execute 221 sec latency 55.838 ms 2024-03-11T20:41:58.160 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:41:58 smithi083 ceph-mon[66365]: pgmap v18: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s 2024-03-11T20:41:58.284 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:41:58 smithi204 ceph-mon[53271]: pgmap v18: 97 pgs: 97 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s 2024-03-11T20:41:59.025 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170132 11.55 MB/sec execute 222 sec latency 72.683 ms 2024-03-11T20:42:00.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170447 11.55 MB/sec execute 223 sec latency 50.127 ms 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 730 op/s 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:42:00.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:42:00.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 730 op/s 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:42:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:01.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170697 11.52 MB/sec execute 224 sec latency 68.210 ms 2024-03-11T20:42:01.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:01 smithi083 ceph-mon[66365]: Reconfiguring mon.smithi083 (monmap changed)... 2024-03-11T20:42:01.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:01 smithi083 ceph-mon[66365]: Reconfiguring daemon mon.smithi083 on smithi083 2024-03-11T20:42:01.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:01 smithi204 ceph-mon[53271]: Reconfiguring mon.smithi083 (monmap changed)... 2024-03-11T20:42:01.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:01 smithi204 ceph-mon[53271]: Reconfiguring daemon mon.smithi083 on smithi083 2024-03-11T20:42:02.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170908 11.48 MB/sec execute 225 sec latency 43.909 ms 2024-03-11T20:42:02.232 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:02 smithi083 ceph-mon[66365]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.4 MiB/s wr, 367 op/s 2024-03-11T20:42:02.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:02 smithi204 ceph-mon[53271]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.4 MiB/s wr, 367 op/s 2024-03-11T20:42:03.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171089 11.45 MB/sec execute 226 sec latency 63.995 ms 2024-03-11T20:42:03.256 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:03.256 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:03.256 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:42:03.256 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:42:03.256 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:03.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:03.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:03.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.jyxogi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:42:03.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:42:03.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:04.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171307 11.42 MB/sec execute 227 sec latency 58.078 ms 2024-03-11T20:42:04.289 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:04 smithi083 ceph-mon[66365]: Reconfiguring mgr.smithi083.jyxogi (monmap changed)... 2024-03-11T20:42:04.289 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:04 smithi083 ceph-mon[66365]: Reconfiguring daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:42:04.289 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:04 smithi083 ceph-mon[66365]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.2 MiB/s wr, 528 op/s 2024-03-11T20:42:04.427 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:04 smithi204 ceph-mon[53271]: Reconfiguring mgr.smithi083.jyxogi (monmap changed)... 2024-03-11T20:42:04.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:04 smithi204 ceph-mon[53271]: Reconfiguring daemon mgr.smithi083.jyxogi on smithi083 2024-03-11T20:42:04.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:04 smithi204 ceph-mon[53271]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.2 MiB/s wr, 528 op/s 2024-03-11T20:42:05.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171570 11.40 MB/sec execute 228 sec latency 70.054 ms 2024-03-11T20:42:06.027 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171884 11.36 MB/sec execute 229 sec latency 66.700 ms 2024-03-11T20:42:06.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-11T20:42:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: Reconfiguring crash.smithi083 (monmap changed)... 2024-03-11T20:42:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:05 smithi083 ceph-mon[66365]: Reconfiguring daemon crash.smithi083 on smithi083 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: Reconfiguring crash.smithi083 (monmap changed)... 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:05 smithi204 ceph-mon[53271]: Reconfiguring daemon crash.smithi083 on smithi083 2024-03-11T20:42:07.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 172332 11.35 MB/sec execute 230 sec latency 42.802 ms 2024-03-11T20:42:08.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 172764 11.37 MB/sec execute 231 sec latency 37.495 ms 2024-03-11T20:42:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:08 smithi083 ceph-mon[66365]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-11T20:42:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:42:08.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:08 smithi204 ceph-mon[53271]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-03-11T20:42:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:42:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:09.027 INFO:teuthology.orchestra.run.smithi083.stdout: 5 173255 11.39 MB/sec execute 232 sec latency 48.391 ms 2024-03-11T20:42:09.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:09 smithi083 ceph-mon[66365]: Reconfiguring osd.0 (monmap changed)... 2024-03-11T20:42:09.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:09 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.0 on smithi083 2024-03-11T20:42:09.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:09 smithi204 ceph-mon[53271]: Reconfiguring osd.0 (monmap changed)... 2024-03-11T20:42:09.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:09 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.0 on smithi083 2024-03-11T20:42:10.026 INFO:teuthology.orchestra.run.smithi083.stdout: 5 173633 11.41 MB/sec execute 233 sec latency 41.106 ms 2024-03-11T20:42:10.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:10 smithi083 ceph-mon[66365]: pgmap v24: 97 pgs: 97 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s 2024-03-11T20:42:10.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:10 smithi204 ceph-mon[53271]: pgmap v24: 97 pgs: 97 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s 2024-03-11T20:42:11.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 173884 11.40 MB/sec execute 234 sec latency 64.840 ms 2024-03-11T20:42:11.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:11.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:11.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: Reconfiguring osd.1 (monmap changed)... 2024-03-11T20:42:11.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:42:11.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:11.430 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.1 on smithi083 2024-03-11T20:42:11.430 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:11 smithi083 ceph-mon[66365]: pgmap v25: 97 pgs: 97 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 367 op/s 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: Reconfiguring osd.1 (monmap changed)... 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.1 on smithi083 2024-03-11T20:42:11.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:11 smithi204 ceph-mon[53271]: pgmap v25: 97 pgs: 97 active+clean; 325 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 367 op/s 2024-03-11T20:42:12.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174144 11.38 MB/sec execute 235 sec latency 43.045 ms 2024-03-11T20:42:12.435 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:12.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:13.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174422 11.35 MB/sec execute 236 sec latency 66.979 ms 2024-03-11T20:42:13.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 563 op/s 2024-03-11T20:42:13.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:13.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:13.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: Reconfiguring osd.2 (monmap changed)... 2024-03-11T20:42:13.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:42:13.967 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:13.967 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:13 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.2 on smithi083 2024-03-11T20:42:14.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174638 11.32 MB/sec execute 237 sec latency 46.883 ms 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 563 op/s 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: Reconfiguring osd.2 (monmap changed)... 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:13 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.2 on smithi083 2024-03-11T20:42:15.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174924 11.31 MB/sec execute 238 sec latency 51.505 ms 2024-03-11T20:42:15.984 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:15 smithi083 ceph-mon[66365]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-03-11T20:42:15.984 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:15 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:15.984 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:15 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:15.985 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:15 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:42:15.985 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:15 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:16.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 175193 11.28 MB/sec execute 239 sec latency 63.953 ms 2024-03-11T20:42:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:15 smithi204 ceph-mon[53271]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-03-11T20:42:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:15 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:15 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:15 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:42:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:15 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:16.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:16 smithi083 ceph-mon[66365]: Reconfiguring osd.3 (monmap changed)... 2024-03-11T20:42:16.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:16 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.3 on smithi083 2024-03-11T20:42:17.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 175843 11.27 MB/sec execute 240 sec latency 28.303 ms 2024-03-11T20:42:17.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:16 smithi204 ceph-mon[53271]: Reconfiguring osd.3 (monmap changed)... 2024-03-11T20:42:17.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:16 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.3 on smithi083 2024-03-11T20:42:17.999 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:17 smithi083 ceph-mon[66365]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-03-11T20:42:18.029 INFO:teuthology.orchestra.run.smithi083.stdout: 5 176408 11.31 MB/sec execute 241 sec latency 28.399 ms 2024-03-11T20:42:18.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:17 smithi204 ceph-mon[53271]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-03-11T20:42:19.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 176940 11.35 MB/sec execute 242 sec latency 37.418 ms 2024-03-11T20:42:19.130 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:42:19.268 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:19.268 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:19.268 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: Reconfiguring mds.foofs.smithi083.motgne (monmap changed)... 2024-03-11T20:42:19.268 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:42:19.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:19.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: Reconfiguring daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:42:19.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: pgmap v29: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 646 op/s 2024-03-11T20:42:19.269 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:19 smithi083 ceph-mon[66365]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:19.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:19.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: Reconfiguring mds.foofs.smithi083.motgne (monmap changed)... 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: Reconfiguring daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: pgmap v29: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 646 op/s 2024-03-11T20:42:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:19 smithi204 ceph-mon[53271]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:20.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 177399 11.37 MB/sec execute 243 sec latency 35.548 ms 2024-03-11T20:42:20.126 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (3m) 41s ago 12m 17.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (12m) 41s ago 12m 6405k - 17.2.0 e1d6a67b021e c9a0119fc5af 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (11m) 26s ago 11m 6409k - 17.2.0 e1d6a67b021e f8b151e24372 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 41s ago 12m 45.8M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (6m) 41s ago 6m 39.3M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (6m) 26s ago 6m 16.7M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (2m) 41s ago 13m 479M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (86s) 26s ago 11m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (65s) 41s ago 14m 31.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (32s) 26s ago 11m 26.9M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (6m) 41s ago 6m 276M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:42:20.127 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (6m) 26s ago 6m 87.3M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (2m) 41s ago 12m 9009k - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (2m) 26s ago 11m 17.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 41s ago 11m 574M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10m) 41s ago 10m 489M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 41s ago 10m 353M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9m) 41s ago 9m 504M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (9m) 26s ago 9m 529M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (8m) 26s ago 8m 370M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (8m) 26s ago 8m 550M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (8m) 26s ago 8m 458M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:42:20.128 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (93s) 41s ago 11m 57.8M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:42:20.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:20 smithi083 ceph-mon[66365]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:20.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:20 smithi083 ceph-mon[66365]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:20.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:20 smithi204 ceph-mon[53271]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:20.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:20 smithi204 ceph-mon[53271]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:20.670 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:42:20.670 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:42:20.670 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:42:20.670 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 4 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:42:20.671 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:42:21.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 177678 11.35 MB/sec execute 244 sec latency 43.509 ms 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:42:21.186 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-03-11T20:42:21.187 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:42:21.187 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "4/23 daemons upgraded", 2024-03-11T20:42:21.187 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mon daemons", 2024-03-11T20:42:21.187 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:42:21.187 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/254147177' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: pgmap v30: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: Reconfiguring crash.smithi204 (monmap changed)... 2024-03-11T20:42:21.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:21.301 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:21.301 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:21 smithi083 ceph-mon[66365]: Reconfiguring daemon crash.smithi204 on smithi204 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/254147177' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: pgmap v30: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: Reconfiguring crash.smithi204 (monmap changed)... 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:21 smithi204 ceph-mon[53271]: Reconfiguring daemon crash.smithi204 on smithi204 2024-03-11T20:42:21.841 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:42:22.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178037 11.33 MB/sec execute 245 sec latency 48.997 ms 2024-03-11T20:42:22.321 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:22 smithi083 ceph-mon[66365]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:22.321 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:22 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/2593593320' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:42:22.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:22 smithi204 ceph-mon[53271]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:22.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:22 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/2593593320' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:42:23.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178359 11.31 MB/sec execute 246 sec latency 61.364 ms 2024-03-11T20:42:23.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 682 op/s 2024-03-11T20:42:23.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:23.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:23.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: Reconfiguring mgr.smithi204.qknpir (monmap changed)... 2024-03-11T20:42:23.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:42:23.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:42:23.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:23.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:23 smithi083 ceph-mon[66365]: Reconfiguring daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:42:24.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178854 11.31 MB/sec execute 247 sec latency 42.581 ms 2024-03-11T20:42:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 682 op/s 2024-03-11T20:42:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: Reconfiguring mgr.smithi204.qknpir (monmap changed)... 2024-03-11T20:42:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi204.qknpir", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-11T20:42:24.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-11T20:42:24.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:24.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:23 smithi204 ceph-mon[53271]: Reconfiguring daemon mgr.smithi204.qknpir on smithi204 2024-03-11T20:42:25.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 179409 11.30 MB/sec execute 248 sec latency 56.867 ms 2024-03-11T20:42:25.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:25.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:25.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: Reconfiguring mon.smithi204 (monmap changed)... 2024-03-11T20:42:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:42:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:42:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: Reconfiguring daemon mon.smithi204 on smithi204 2024-03-11T20:42:25.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:25 smithi083 ceph-mon[66365]: pgmap v32: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-03-11T20:42:25.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:25.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:25.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: Reconfiguring mon.smithi204 (monmap changed)... 2024-03-11T20:42:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-11T20:42:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-11T20:42:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: Reconfiguring daemon mon.smithi204 on smithi204 2024-03-11T20:42:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:25 smithi204 ceph-mon[53271]: pgmap v32: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-03-11T20:42:26.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 179891 11.32 MB/sec execute 249 sec latency 30.828 ms 2024-03-11T20:42:27.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 180437 11.35 MB/sec execute 250 sec latency 35.967 ms 2024-03-11T20:42:28.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 180877 11.38 MB/sec execute 251 sec latency 32.975 ms 2024-03-11T20:42:28.068 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: pgmap v33: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-03-11T20:42:28.068 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:28.068 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:28.068 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:28.068 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: Reconfiguring osd.4 (monmap changed)... 2024-03-11T20:42:28.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:42:28.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:28.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:27 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.4 on smithi204 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: pgmap v33: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: Reconfiguring osd.4 (monmap changed)... 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:27 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.4 on smithi204 2024-03-11T20:42:29.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 181220 11.37 MB/sec execute 252 sec latency 31.215 ms 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: Reconfiguring osd.5 (monmap changed)... 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.5 on smithi204 2024-03-11T20:42:29.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:29 smithi083 ceph-mon[66365]: pgmap v34: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 784 op/s 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: Reconfiguring osd.5 (monmap changed)... 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.5 on smithi204 2024-03-11T20:42:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:29 smithi204 ceph-mon[53271]: pgmap v34: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 784 op/s 2024-03-11T20:42:30.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 181667 11.35 MB/sec execute 253 sec latency 39.569 ms 2024-03-11T20:42:31.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182115 11.35 MB/sec execute 254 sec latency 38.577 ms 2024-03-11T20:42:31.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:31.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:31.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: Reconfiguring osd.6 (monmap changed)... 2024-03-11T20:42:31.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:42:31.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:31.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.6 on smithi204 2024-03-11T20:42:31.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:31 smithi083 ceph-mon[66365]: pgmap v35: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 540 op/s 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: Reconfiguring osd.6 (monmap changed)... 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.6 on smithi204 2024-03-11T20:42:31.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:31 smithi204 ceph-mon[53271]: pgmap v35: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 540 op/s 2024-03-11T20:42:32.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182736 11.35 MB/sec execute 255 sec latency 38.669 ms 2024-03-11T20:42:33.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 183294 11.36 MB/sec execute 256 sec latency 32.157 ms 2024-03-11T20:42:33.653 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:33.653 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:33.653 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: Reconfiguring osd.7 (monmap changed)... 2024-03-11T20:42:33.654 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:42:33.654 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:33.654 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: Reconfiguring daemon osd.7 on smithi204 2024-03-11T20:42:33.654 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:33 smithi083 ceph-mon[66365]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 838 op/s 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: Reconfiguring osd.7 (monmap changed)... 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: Reconfiguring daemon osd.7 on smithi204 2024-03-11T20:42:33.678 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:33 smithi204 ceph-mon[53271]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 838 op/s 2024-03-11T20:42:34.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 183676 11.36 MB/sec execute 257 sec latency 43.542 ms 2024-03-11T20:42:35.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184039 11.38 MB/sec execute 258 sec latency 46.363 ms 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: Reconfiguring mds.foofs.smithi204.evhral (monmap changed)... 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: Reconfiguring daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:42:35.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:35 smithi204 ceph-mon[53271]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: Reconfiguring mds.foofs.smithi204.evhral (monmap changed)... 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: Reconfiguring daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:42:35.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:35 smithi083 ceph-mon[66365]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-03-11T20:42:36.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184324 11.38 MB/sec execute 259 sec latency 40.439 ms 2024-03-11T20:42:37.030 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184563 11.37 MB/sec execute 260 sec latency 57.356 ms 2024-03-11T20:42:37.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all mon 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]: dispatch 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]': finished 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi204"}]: dispatch 2024-03-11T20:42:37.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi204"}]': finished 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all mon 2024-03-11T20:42:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]: dispatch 2024-03-11T20:42:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]': finished 2024-03-11T20:42:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi204"}]: dispatch 2024-03-11T20:42:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi204"}]': finished 2024-03-11T20:42:38.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184944 11.36 MB/sec execute 261 sec latency 67.287 ms 2024-03-11T20:42:38.265 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:38 smithi083 ceph-mon[66365]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-03-11T20:42:38.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:38 smithi204 ceph-mon[53271]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-03-11T20:42:39.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185346 11.34 MB/sec execute 262 sec latency 37.073 ms 2024-03-11T20:42:40.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185652 11.33 MB/sec execute 263 sec latency 66.718 ms 2024-03-11T20:42:40.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: pgmap v39: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 799 op/s 2024-03-11T20:42:40.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: Upgrade: Updating crash.smithi083 (1/2) 2024-03-11T20:42:40.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:40.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:40.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:40.048 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:39 smithi083 ceph-mon[66365]: Deploying daemon crash.smithi083 on smithi083 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: pgmap v39: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 799 op/s 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: Upgrade: Updating crash.smithi083 (1/2) 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:39 smithi204 ceph-mon[53271]: Deploying daemon crash.smithi083 on smithi083 2024-03-11T20:42:41.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186022 11.31 MB/sec execute 264 sec latency 48.457 ms 2024-03-11T20:42:42.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186388 11.29 MB/sec execute 265 sec latency 49.061 ms 2024-03-11T20:42:42.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:41 smithi083 ceph-mon[66365]: pgmap v40: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-03-11T20:42:42.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:42.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:41 smithi204 ceph-mon[53271]: pgmap v40: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-03-11T20:42:42.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:43.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186715 11.28 MB/sec execute 266 sec latency 67.669 ms 2024-03-11T20:42:43.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:43 smithi083 ceph-mon[66365]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 735 op/s 2024-03-11T20:42:44.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 187255 11.30 MB/sec execute 267 sec latency 34.794 ms 2024-03-11T20:42:44.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:43 smithi204 ceph-mon[53271]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 735 op/s 2024-03-11T20:42:45.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 187766 11.33 MB/sec execute 268 sec latency 57.831 ms 2024-03-11T20:42:45.889 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:45 smithi204 ceph-mon[53271]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-03-11T20:42:45.889 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:45 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:45.889 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:45 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:45.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:45 smithi083 ceph-mon[66365]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-03-11T20:42:45.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:45 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:45.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:45 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:46.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 188161 11.35 MB/sec execute 269 sec latency 39.346 ms 2024-03-11T20:42:47.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 188581 11.34 MB/sec execute 270 sec latency 36.547 ms 2024-03-11T20:42:47.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: Upgrade: Updating crash.smithi204 (2/2) 2024-03-11T20:42:47.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:47.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:47.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:47.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: Deploying daemon crash.smithi204 on smithi204 2024-03-11T20:42:47.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:47 smithi083 ceph-mon[66365]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: Upgrade: Updating crash.smithi204 (2/2) 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi204", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: Deploying daemon crash.smithi204 on smithi204 2024-03-11T20:42:47.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:47 smithi204 ceph-mon[53271]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-03-11T20:42:48.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 188894 11.31 MB/sec execute 271 sec latency 41.043 ms 2024-03-11T20:42:49.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 189286 11.31 MB/sec execute 272 sec latency 45.822 ms 2024-03-11T20:42:50.017 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:49 smithi083 ceph-mon[66365]: pgmap v44: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-03-11T20:42:50.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 189875 11.30 MB/sec execute 273 sec latency 33.050 ms 2024-03-11T20:42:50.046 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:49 smithi204 ceph-mon[53271]: pgmap v44: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-03-11T20:42:51.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190148 11.28 MB/sec execute 274 sec latency 49.122 ms 2024-03-11T20:42:52.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190424 11.28 MB/sec execute 275 sec latency 63.071 ms 2024-03-11T20:42:52.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:51 smithi083 ceph-mon[66365]: pgmap v45: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-03-11T20:42:52.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:52.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:52.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:42:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:51 smithi204 ceph-mon[53271]: pgmap v45: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-03-11T20:42:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:42:52.330 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:42:53.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190630 11.27 MB/sec execute 276 sec latency 72.116 ms 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (3m) 74s ago 13m 17.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 starting - - - - 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 starting - - - - 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 74s ago 12m 45.8M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (7m) 74s ago 7m 39.3M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (7m) 59s ago 7m 16.7M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:42:53.224 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (2m) 74s ago 14m 479M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (119s) 59s ago 12m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (98s) 74s ago 14m 31.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (66s) 59s ago 12m 26.9M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (6m) 74s ago 6m 276M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (6m) 59s ago 6m 87.3M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (3m) 74s ago 12m 9009k - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (3m) 59s ago 12m 17.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 74s ago 11m 574M 1936M 17.2.0 e1d6a67b021e ff6acb4be6fc 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 74s ago 11m 489M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 74s ago 10m 353M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 74s ago 10m 504M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (9m) 59s ago 9m 529M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:42:53.225 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (9m) 59s ago 9m 370M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:42:53.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (8m) 59s ago 8m 550M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:42:53.226 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (8m) 59s ago 8m 458M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:42:53.226 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (2m) 74s ago 11m 57.8M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:42:53.920 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:42:53.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:42:53.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:42:53.920 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:53.920 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 4 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:42:53.921 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:42:54.031 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190966 11.26 MB/sec execute 277 sec latency 73.936 ms 2024-03-11T20:42:54.088 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:53 smithi083 ceph-mon[66365]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.088 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:53 smithi083 ceph-mon[66365]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:53 smithi083 ceph-mon[66365]: pgmap v46: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 722 op/s 2024-03-11T20:42:54.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:53 smithi204 ceph-mon[53271]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:53 smithi204 ceph-mon[53271]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:53 smithi204 ceph-mon[53271]: pgmap v46: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 722 op/s 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:42:54.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-03-11T20:42:54.374 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:42:54.374 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "4/23 daemons upgraded", 2024-03-11T20:42:54.374 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading crash daemons", 2024-03-11T20:42:54.374 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:42:54.374 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:42:54.872 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:54 smithi083 ceph-mon[66365]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.872 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:54 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/443890597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:54.941 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:42:54.972 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:54 smithi204 ceph-mon[53271]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:54.972 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:54 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/443890597' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:42:55.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 191230 11.26 MB/sec execute 278 sec latency 53.341 ms 2024-03-11T20:42:55.920 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:55 smithi083 ceph-mon[66365]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:55.920 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:55 smithi083 ceph-mon[66365]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-03-11T20:42:55.920 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:55 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1706012193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:42:56.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 191528 11.27 MB/sec execute 279 sec latency 53.269 ms 2024-03-11T20:42:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:55 smithi204 ceph-mon[53271]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:42:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:55 smithi204 ceph-mon[53271]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-03-11T20:42:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:55 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1706012193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:42:57.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 191759 11.25 MB/sec execute 280 sec latency 51.982 ms 2024-03-11T20:42:57.958 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:57 smithi083 ceph-mon[66365]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-03-11T20:42:57.959 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:57.959 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:57.959 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192007 11.23 MB/sec execute 281 sec latency 60.078 ms 2024-03-11T20:42:58.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:57 smithi204 ceph-mon[53271]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-03-11T20:42:58.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:42:58.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:58 smithi083 ceph-mon[66365]: mgrmap e43: smithi083.jyxogi(active, since 91s), standbys: smithi204.qknpir 2024-03-11T20:42:58.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:58 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.966 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:42:58 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.978 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:58 smithi204 ceph-mon[53271]: mgrmap e43: smithi083.jyxogi(active, since 91s), standbys: smithi204.qknpir 2024-03-11T20:42:58.979 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:58 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:58.979 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:42:58 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:42:59.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192285 11.20 MB/sec execute 282 sec latency 64.670 ms 2024-03-11T20:43:00.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192638 11.20 MB/sec execute 283 sec latency 38.488 ms 2024-03-11T20:43:00.225 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:00 smithi083 ceph-mon[66365]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 644 op/s 2024-03-11T20:43:00.226 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:00.226 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:00.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:00 smithi204 ceph-mon[53271]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 644 op/s 2024-03-11T20:43:00.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:00.415 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:01.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192991 11.18 MB/sec execute 284 sec latency 47.244 ms 2024-03-11T20:43:02.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193381 11.16 MB/sec execute 285 sec latency 51.053 ms 2024-03-11T20:43:02.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:02 smithi204 ceph-mon[53271]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 382 op/s 2024-03-11T20:43:02.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:02 smithi083 ceph-mon[66365]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 382 op/s 2024-03-11T20:43:02.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:02.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:03.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193700 11.14 MB/sec execute 286 sec latency 39.227 ms 2024-03-11T20:43:04.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194026 11.15 MB/sec execute 287 sec latency 62.218 ms 2024-03-11T20:43:04.318 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:04 smithi083 ceph-mon[66365]: pgmap v51: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 570 op/s 2024-03-11T20:43:04.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:04 smithi204 ceph-mon[53271]: pgmap v51: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 570 op/s 2024-03-11T20:43:05.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194464 11.16 MB/sec execute 288 sec latency 35.670 ms 2024-03-11T20:43:06.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194884 11.18 MB/sec execute 289 sec latency 49.789 ms 2024-03-11T20:43:06.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:06 smithi083 ceph-mon[66365]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-03-11T20:43:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:06 smithi204 ceph-mon[53271]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-03-11T20:43:07.032 INFO:teuthology.orchestra.run.smithi083.stdout: 5 195350 11.19 MB/sec execute 290 sec latency 34.025 ms 2024-03-11T20:43:08.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 195830 11.18 MB/sec execute 291 sec latency 24.474 ms 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:08.136 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]: dispatch 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]': finished 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi204"}]: dispatch 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi204"}]': finished 2024-03-11T20:43:08.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-11T20:43:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-03-11T20:43:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]': finished 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi204"}]: dispatch 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi204"}]': finished 2024-03-11T20:43:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-11T20:43:09.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 196221 11.18 MB/sec execute 292 sec latency 29.658 ms 2024-03-11T20:43:09.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:09 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all crash 2024-03-11T20:43:09.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:09 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-11T20:43:09.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:09 smithi083 ceph-mon[66365]: Upgrade: osd.0 is safe to restart 2024-03-11T20:43:09.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:09 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all crash 2024-03-11T20:43:09.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:09 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-11T20:43:09.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:09 smithi204 ceph-mon[53271]: Upgrade: osd.0 is safe to restart 2024-03-11T20:43:10.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 196695 11.17 MB/sec execute 293 sec latency 56.433 ms 2024-03-11T20:43:10.173 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:10 smithi083 ceph-mon[66365]: pgmap v54: 97 pgs: 97 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 623 op/s 2024-03-11T20:43:10.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:10 smithi204 ceph-mon[53271]: pgmap v54: 97 pgs: 97 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 623 op/s 2024-03-11T20:43:11.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 197152 11.15 MB/sec execute 294 sec latency 37.123 ms 2024-03-11T20:43:11.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: Upgrade: Updating osd.0 2024-03-11T20:43:11.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: pgmap v55: 97 pgs: 97 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 467 op/s 2024-03-11T20:43:11.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:11.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:43:11.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:11.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:11 smithi083 ceph-mon[66365]: Deploying daemon osd.0 on smithi083 2024-03-11T20:43:12.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 197540 11.17 MB/sec execute 295 sec latency 58.808 ms 2024-03-11T20:43:12.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: Upgrade: Updating osd.0 2024-03-11T20:43:12.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: pgmap v55: 97 pgs: 97 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 467 op/s 2024-03-11T20:43:12.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:12.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-11T20:43:12.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:12.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:11 smithi204 ceph-mon[53271]: Deploying daemon osd.0 on smithi083 2024-03-11T20:43:13.000 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:13.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 197971 11.17 MB/sec execute 296 sec latency 40.009 ms 2024-03-11T20:43:13.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:14.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:13 smithi083 ceph-mon[66365]: pgmap v56: 97 pgs: 97 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 595 op/s 2024-03-11T20:43:14.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 198373 11.19 MB/sec execute 297 sec latency 47.574 ms 2024-03-11T20:43:14.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:13 smithi204 ceph-mon[53271]: pgmap v56: 97 pgs: 97 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 595 op/s 2024-03-11T20:43:15.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 198733 11.20 MB/sec execute 298 sec latency 41.513 ms 2024-03-11T20:43:16.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199069 11.19 MB/sec execute 299 sec latency 38.557 ms 2024-03-11T20:43:16.040 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:15 smithi083 ceph-mon[66365]: pgmap v57: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 544 op/s 2024-03-11T20:43:16.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:15 smithi204 ceph-mon[53271]: pgmap v57: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 544 op/s 2024-03-11T20:43:16.548 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:16 smithi083 systemd[1]: Stopping Ceph osd.0 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:43:16.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:16 smithi083 ceph-mon[66365]: osd.0 marked itself down and dead 2024-03-11T20:43:16.810 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:16 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[34880]: 2024-03-11T20:43:16.762+0000 7f13b2056700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:43:16.810 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:16 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[34880]: 2024-03-11T20:43:16.762+0000 7f13b2056700 -1 osd.0 73 *** Got signal Terminated *** 2024-03-11T20:43:16.810 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:16 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[34880]: 2024-03-11T20:43:16.762+0000 7f13b2056700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:43:17.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199235 11.16 MB/sec execute 300 sec latency 235.897 ms 2024-03-11T20:43:17.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:16 smithi204 ceph-mon[53271]: osd.0 marked itself down and dead 2024-03-11T20:43:17.808 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:17 smithi083 podman[71896]: 2024-03-11 20:43:17.45775516 +0000 UTC m=+0.813825793 container died ff6acb4be6fcd005d17d50a911ec44c7518aace758498d99add6dc7326542b73 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.tags=base centos centos-stream, release=754, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-03-11T20:43:18.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199285 11.13 MB/sec execute 301 sec latency 1063.365 ms 2024-03-11T20:43:18.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:17 smithi083 ceph-mon[66365]: pgmap v58: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-03-11T20:43:18.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:17 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:43:18.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:17 smithi083 ceph-mon[66365]: osdmap e74: 8 total, 7 up, 8 in 2024-03-11T20:43:18.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:17 smithi204 ceph-mon[53271]: pgmap v58: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-03-11T20:43:18.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:17 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:43:18.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:17 smithi204 ceph-mon[53271]: osdmap e74: 8 total, 7 up, 8 in 2024-03-11T20:43:18.567 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:18 smithi083 podman[71896]: 2024-03-11 20:43:18.490474881 +0000 UTC m=+1.846545515 container cleanup ff6acb4be6fcd005d17d50a911ec44c7518aace758498d99add6dc7326542b73 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, ceph=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, RELEASE=HEAD, version=8) 2024-03-11T20:43:18.567 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:18 smithi083 bash[71896]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0 2024-03-11T20:43:18.818 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:18 smithi083 ceph-mon[66365]: osdmap e75: 8 total, 7 up, 8 in 2024-03-11T20:43:19.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199644 11.11 MB/sec execute 302 sec latency 67.243 ms 2024-03-11T20:43:19.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:18 smithi204 ceph-mon[53271]: osdmap e75: 8 total, 7 up, 8 in 2024-03-11T20:43:19.320 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:19 smithi083 podman[71912]: 2024-03-11 20:43:19.123831426 +0000 UTC m=+1.668351181 container remove ff6acb4be6fcd005d17d50a911ec44c7518aace758498d99add6dc7326542b73 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, ceph=True) 2024-03-11T20:43:19.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:19 smithi083 ceph-mon[66365]: pgmap v61: 97 pgs: 36 peering, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 648 op/s 2024-03-11T20:43:19.825 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:19 smithi083 ceph-mon[66365]: Health check failed: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-03-11T20:43:19.825 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:19 smithi083 podman[72007]: 2024-03-11 20:43:19.594834593 +0000 UTC m=+0.025001474 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:43:19.825 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:19 smithi083 podman[72007]: 2024-03-11 20:43:19.742071302 +0000 UTC m=+0.172238156 container create ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-11T20:43:20.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199986 11.10 MB/sec execute 303 sec latency 61.970 ms 2024-03-11T20:43:20.087 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:20 smithi083 podman[72007]: 2024-03-11 20:43:20.028567057 +0000 UTC m=+0.458733919 container init ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e) 2024-03-11T20:43:20.088 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:20 smithi083 podman[72007]: 2024-03-11 20:43:20.033302437 +0000 UTC m=+0.463469300 container start ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, RELEASE=main-2b0101e) 2024-03-11T20:43:20.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:19 smithi204 ceph-mon[53271]: pgmap v61: 97 pgs: 36 peering, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 648 op/s 2024-03-11T20:43:20.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:19 smithi204 ceph-mon[53271]: Health check failed: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-03-11T20:43:20.339 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:20 smithi083 podman[72007]: 2024-03-11 20:43:20.14712544 +0000 UTC m=+0.577292318 container attach ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307) 2024-03-11T20:43:20.603 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:20 smithi083 podman[72007]: 2024-03-11 20:43:20.45023328 +0000 UTC m=+0.880400134 container died ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD) 2024-03-11T20:43:21.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 200362 11.09 MB/sec execute 304 sec latency 82.572 ms 2024-03-11T20:43:21.643 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 podman[72032]: 2024-03-11 20:43:21.420172472 +0000 UTC m=+0.956640628 container remove ad0643c3e95ff0ab38ed7bd244f040405f3e1f924d974fc2cf126fbeec0371a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, RELEASE=main-2b0101e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:43:21.644 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-11T20:43:21.644 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 systemd[1]: Stopped Ceph osd.0 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:43:21.644 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.0.service: Consumed 3min 38.676s CPU time. 2024-03-11T20:43:21.809 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 systemd[1]: Starting Ceph osd.0 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:43:21.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:21 smithi083 ceph-mon[66365]: pgmap v62: 97 pgs: 36 peering, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-03-11T20:43:22.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 200666 11.07 MB/sec execute 305 sec latency 60.432 ms 2024-03-11T20:43:22.146 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:21 smithi083 podman[72164]: 2024-03-11 20:43:21.892518302 +0000 UTC m=+0.033583975 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:43:22.147 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 podman[72164]: 2024-03-11 20:43:22.047495524 +0000 UTC m=+0.188561189 container create 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-03-11T20:43:22.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:21 smithi204 ceph-mon[53271]: pgmap v62: 97 pgs: 36 peering, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-03-11T20:43:22.419 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 podman[72164]: 2024-03-11 20:43:22.385318052 +0000 UTC m=+0.526383726 container init 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True) 2024-03-11T20:43:22.419 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 podman[72164]: 2024-03-11 20:43:22.390203603 +0000 UTC m=+0.531269276 container start 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-11T20:43:22.671 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 podman[72164]: 2024-03-11 20:43:22.537968021 +0000 UTC m=+0.679033690 container attach 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-11T20:43:22.926 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 224/1551 objects degraded (14.442%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:22.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-mon[66365]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 bash[72164]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-11T20:43:22.927 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-11T20:43:22.928 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-11T20:43:22.928 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-11T20:43:22.928 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate[72179]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-11T20:43:23.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 200939 11.05 MB/sec execute 306 sec latency 62.738 ms 2024-03-11T20:43:23.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:22 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 224/1551 objects degraded (14.442%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:23.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:22 smithi204 ceph-mon[53271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-03-11T20:43:23.185 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:22 smithi083 podman[72164]: 2024-03-11 20:43:22.966673662 +0000 UTC m=+1.107739343 container died 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-11T20:43:23.967 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:23 smithi083 ceph-mon[66365]: pgmap v63: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 575 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:24.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 201262 11.05 MB/sec execute 307 sec latency 56.765 ms 2024-03-11T20:43:24.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:23 smithi204 ceph-mon[53271]: pgmap v63: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 575 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:24.217 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:24 smithi083 podman[72311]: 2024-03-11 20:43:24.210970886 +0000 UTC m=+1.238042061 container remove 007e4b8f3578e068f123aa566d34e1c8ee8f1194eb1a96b685675c6d8e249483 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-11T20:43:24.719 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:24 smithi083 podman[72367]: 2024-03-11 20:43:24.382503508 +0000 UTC m=+0.027278742 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:43:24.719 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:24 smithi083 podman[72367]: 2024-03-11 20:43:24.711860943 +0000 UTC m=+0.356636169 container create ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-11T20:43:25.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 201939 11.10 MB/sec execute 308 sec latency 21.655 ms 2024-03-11T20:43:25.222 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 podman[72367]: 2024-03-11 20:43:25.106357136 +0000 UTC m=+0.751132372 container init ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307) 2024-03-11T20:43:25.223 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 podman[72367]: 2024-03-11 20:43:25.116563385 +0000 UTC m=+0.761338614 container start ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-11T20:43:25.223 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-osd[72408]: -- 172.21.15.83:0/1586137099 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556b12111680 con 0x556b120dac00 2024-03-11T20:43:25.318 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 bash[72367]: ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 2024-03-11T20:43:25.318 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 systemd[1]: Started Ceph osd.0 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:43:25.542 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:43:25.828 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:43:25.691+0000 7f77e3dbc7c0 -1 Falling back to public interface 2024-03-11T20:43:26.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 202341 11.12 MB/sec execute 309 sec latency 43.697 ms 2024-03-11T20:43:26.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-mon[66365]: pgmap v64: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 575 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:26.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:26.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:26.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:26.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:25 smithi204 ceph-mon[53271]: pgmap v64: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 575 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:26.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:26.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:26.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 26s ago 13m 17.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (42s) 26s ago 13m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (35s) 29s ago 13m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (4m) 26s ago 13m 46.8M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (7m) 26s ago 7m 39.0M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (7m) 29s ago 7m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (3m) 26s ago 15m 497M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (2m) 29s ago 13m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:43:26.568 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (2m) 26s ago 15m 42.5M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (99s) 29s ago 12m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (7m) 26s ago 7m 282M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (7m) 29s ago 7m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (4m) 26s ago 13m 13.4M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (3m) 29s ago 12m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 starting - - - 1936M 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 26s ago 11m 579M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 26s ago 11m 467M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 26s ago 10m 603M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (10m) 29s ago 10m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (9m) 29s ago 9m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (9m) 29s ago 9m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (9m) 29s ago 9m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:43:26.569 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (2m) 26s ago 12m 67.5M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:43:27.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 202736 11.11 MB/sec execute 310 sec latency 49.654 ms 2024-03-11T20:43:27.096 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:26 smithi083 ceph-mon[66365]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:27.096 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:26 smithi083 ceph-mon[66365]: from='client.44149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:27.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:26 smithi204 ceph-mon[53271]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:27.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:26 smithi204 ceph-mon[53271]: from='client.44149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:27.213 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:43:27.214 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:43:27.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-11T20:43:27.215 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 4 2024-03-11T20:43:27.215 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:43:27.215 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:43:27.842 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "6/23 daemons upgraded", 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:43:27.843 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:43:28.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203179 11.09 MB/sec execute 311 sec latency 39.393 ms 2024-03-11T20:43:28.107 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:27 smithi083 ceph-mon[66365]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:28.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:27 smithi083 ceph-mon[66365]: pgmap v65: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:28.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:28.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:28.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:27 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1684507376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:27 smithi204 ceph-mon[53271]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:27 smithi204 ceph-mon[53271]: pgmap v65: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 330 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 224/1551 objects degraded (14.442%) 2024-03-11T20:43:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:28.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:27 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1684507376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 224/1551 objects degraded (14.442%), 28 pgs degraded 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: osd.0 (root=default,host=smithi083) is down 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 224/1551 objects degraded (14.442%), 28 pgs degraded 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-03-11T20:43:28.562 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.8 is active+undersized+degraded, acting [2,1] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-03-11T20:43:28.563 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-03-11T20:43:28.564 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-03-11T20:43:29.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203548 11.09 MB/sec execute 312 sec latency 48.340 ms 2024-03-11T20:43:29.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:28 smithi083 ceph-mon[66365]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:29.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:28 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/704297190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:43:29.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:28 smithi204 ceph-mon[53271]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:43:29.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:28 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/704297190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:43:29.892 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:29 smithi083 ceph-mon[66365]: pgmap v66: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 518 op/s; 246/1770 objects degraded (13.898%) 2024-03-11T20:43:29.892 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:29 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 246/1770 objects degraded (13.898%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:30.033 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204012 11.08 MB/sec execute 313 sec latency 45.751 ms 2024-03-11T20:43:30.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:29 smithi204 ceph-mon[53271]: pgmap v66: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 518 op/s; 246/1770 objects degraded (13.898%) 2024-03-11T20:43:30.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:29 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 246/1770 objects degraded (13.898%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:31.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204366 11.07 MB/sec execute 314 sec latency 39.185 ms 2024-03-11T20:43:31.919 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:31 smithi083 ceph-mon[66365]: pgmap v67: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 246/1770 objects degraded (13.898%) 2024-03-11T20:43:32.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204703 11.07 MB/sec execute 315 sec latency 61.094 ms 2024-03-11T20:43:32.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:31 smithi204 ceph-mon[53271]: pgmap v67: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 246/1770 objects degraded (13.898%) 2024-03-11T20:43:32.944 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:32 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:43:32.800+0000 7f77e3dbc7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-03-11T20:43:33.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 205160 11.09 MB/sec execute 316 sec latency 40.239 ms 2024-03-11T20:43:33.960 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:33 smithi083 ceph-mon[66365]: pgmap v68: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 714 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:34.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 205648 11.12 MB/sec execute 317 sec latency 29.307 ms 2024-03-11T20:43:34.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:33 smithi204 ceph-mon[53271]: pgmap v68: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 714 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:34.472 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:34 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:43:34.312+0000 7f77e3dbc7c0 -1 osd.0 73 log_to_monitors true 2024-03-11T20:43:35.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 205947 11.11 MB/sec execute 318 sec latency 37.352 ms 2024-03-11T20:43:35.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:35.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:35.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:35 smithi204 ceph-mon[53271]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-11T20:43:35.238 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:35.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:35.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:35 smithi083 ceph-mon[66365]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-11T20:43:35.743 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:43:35 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:43:35.608+0000 7f77d120c700 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:43:36.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206322 11.10 MB/sec execute 319 sec latency 48.980 ms 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: pgmap v69: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 497 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: osdmap e76: 8 total, 7 up, 8 in 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 234/1644 objects degraded (14.234%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:36.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:36 smithi083 ceph-mon[66365]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' 2024-03-11T20:43:36.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: pgmap v69: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 497 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:36.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-11T20:43:36.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: osdmap e76: 8 total, 7 up, 8 in 2024-03-11T20:43:36.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:43:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 234/1644 objects degraded (14.234%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:36 smithi204 ceph-mon[53271]: from='osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835]' entity='osd.0' 2024-03-11T20:43:37.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206497 11.07 MB/sec execute 320 sec latency 436.495 ms 2024-03-11T20:43:37.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:37 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:43:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:37 smithi204 ceph-mon[53271]: osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835] boot 2024-03-11T20:43:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:37 smithi204 ceph-mon[53271]: osdmap e77: 8 total, 8 up, 8 in 2024-03-11T20:43:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:43:37.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:37 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:43:37.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:37 smithi083 ceph-mon[66365]: osd.0 [v2:172.21.15.83:6802/2628905835,v1:172.21.15.83:6803/2628905835] boot 2024-03-11T20:43:37.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:37 smithi083 ceph-mon[66365]: osdmap e77: 8 total, 8 up, 8 in 2024-03-11T20:43:37.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-11T20:43:38.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206497 11.03 MB/sec execute 321 sec latency 1436.521 ms 2024-03-11T20:43:38.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:38 smithi204 ceph-mon[53271]: pgmap v72: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.1 MiB/s wr, 359 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:38.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:38 smithi204 ceph-mon[53271]: osdmap e78: 8 total, 8 up, 8 in 2024-03-11T20:43:38.445 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:38 smithi083 ceph-mon[66365]: pgmap v72: 97 pgs: 28 active+undersized+degraded, 8 active+undersized, 61 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.1 MiB/s wr, 359 op/s; 234/1644 objects degraded (14.234%) 2024-03-11T20:43:38.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:38 smithi083 ceph-mon[66365]: osdmap e78: 8 total, 8 up, 8 in 2024-03-11T20:43:39.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206497 11.00 MB/sec execute 322 sec latency 2436.548 ms 2024-03-11T20:43:39.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:39 smithi083 ceph-mon[66365]: osdmap e79: 8 total, 8 up, 8 in 2024-03-11T20:43:39.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:39 smithi083 ceph-mon[66365]: pgmap v75: 97 pgs: 12 remapped+peering, 1 active+recovering+degraded, 3 active+undersized+degraded, 5 active+undersized, 76 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 540 op/s; 23/1707 objects degraded (1.347%); 31 KiB/s, 1 objects/s recovering 2024-03-11T20:43:39.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:39 smithi204 ceph-mon[53271]: osdmap e79: 8 total, 8 up, 8 in 2024-03-11T20:43:39.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:39 smithi204 ceph-mon[53271]: pgmap v75: 97 pgs: 12 remapped+peering, 1 active+recovering+degraded, 3 active+undersized+degraded, 5 active+undersized, 76 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 540 op/s; 23/1707 objects degraded (1.347%); 31 KiB/s, 1 objects/s recovering 2024-03-11T20:43:40.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206700 10.97 MB/sec execute 323 sec latency 2646.593 ms 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: osdmap e80: 8 total, 8 up, 8 in 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 23/1707 objects degraded (1.347%), 4 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: osdmap e80: 8 total, 8 up, 8 in 2024-03-11T20:43:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 23/1707 objects degraded (1.347%), 4 pgs degraded (PG_DEGRADED) 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:43:40.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:41.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207071 10.98 MB/sec execute 324 sec latency 465.015 ms 2024-03-11T20:43:41.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:41 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:41.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:41 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-11T20:43:41.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:41 smithi083 ceph-mon[66365]: osdmap e81: 8 total, 8 up, 8 in 2024-03-11T20:43:41.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:41 smithi083 ceph-mon[66365]: pgmap v78: 97 pgs: 10 remapped+peering, 2 peering, 1 active+recovering+degraded, 84 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 551 op/s; 1/1680 objects degraded (0.060%); 31 KiB/s, 13 objects/s recovering 2024-03-11T20:43:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:41 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:41 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-11T20:43:41.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:41 smithi204 ceph-mon[53271]: osdmap e81: 8 total, 8 up, 8 in 2024-03-11T20:43:41.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:41 smithi204 ceph-mon[53271]: pgmap v78: 97 pgs: 10 remapped+peering, 2 peering, 1 active+recovering+degraded, 84 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 551 op/s; 1/1680 objects degraded (0.060%); 31 KiB/s, 13 objects/s recovering 2024-03-11T20:43:42.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207466 10.97 MB/sec execute 325 sec latency 398.154 ms 2024-03-11T20:43:42.891 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:42 smithi083 ceph-mon[66365]: osdmap e82: 8 total, 8 up, 8 in 2024-03-11T20:43:42.892 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:42.892 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:42.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:42 smithi204 ceph-mon[53271]: osdmap e82: 8 total, 8 up, 8 in 2024-03-11T20:43:42.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:42.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:43.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207674 10.94 MB/sec execute 326 sec latency 1034.250 ms 2024-03-11T20:43:43.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:43 smithi083 ceph-mon[66365]: osdmap e83: 8 total, 8 up, 8 in 2024-03-11T20:43:43.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:43 smithi083 ceph-mon[66365]: pgmap v81: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.9 MiB/s wr, 384 op/s; 1.5 MiB/s, 50 objects/s recovering 2024-03-11T20:43:43.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:43 smithi204 ceph-mon[53271]: osdmap e83: 8 total, 8 up, 8 in 2024-03-11T20:43:43.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:43 smithi204 ceph-mon[53271]: pgmap v81: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.9 MiB/s wr, 384 op/s; 1.5 MiB/s, 50 objects/s recovering 2024-03-11T20:43:44.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208070 10.92 MB/sec execute 327 sec latency 44.075 ms 2024-03-11T20:43:44.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:44 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/1680 objects degraded (0.060%), 1 pg degraded) 2024-03-11T20:43:44.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:44 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:43:44.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:44 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/1680 objects degraded (0.060%), 1 pg degraded) 2024-03-11T20:43:44.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:44 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:43:45.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208305 10.92 MB/sec execute 328 sec latency 49.431 ms 2024-03-11T20:43:45.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:45 smithi083 ceph-mon[66365]: pgmap v82: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 296 op/s; 1.2 MiB/s, 39 objects/s recovering 2024-03-11T20:43:45.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:45 smithi204 ceph-mon[53271]: pgmap v82: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 296 op/s; 1.2 MiB/s, 39 objects/s recovering 2024-03-11T20:43:46.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208576 10.92 MB/sec execute 329 sec latency 49.314 ms 2024-03-11T20:43:47.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208914 10.93 MB/sec execute 330 sec latency 83.051 ms 2024-03-11T20:43:48.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209221 10.94 MB/sec execute 331 sec latency 68.660 ms 2024-03-11T20:43:48.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:47 smithi083 ceph-mon[66365]: pgmap v83: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.0 MiB/s wr, 274 op/s; 1.0 MiB/s, 42 objects/s recovering 2024-03-11T20:43:48.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:47 smithi204 ceph-mon[53271]: pgmap v83: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.0 MiB/s wr, 274 op/s; 1.0 MiB/s, 42 objects/s recovering 2024-03-11T20:43:49.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209463 10.93 MB/sec execute 332 sec latency 70.768 ms 2024-03-11T20:43:49.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:49 smithi083 ceph-mon[66365]: pgmap v84: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 791 KiB/s, 33 objects/s recovering 2024-03-11T20:43:50.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209665 10.91 MB/sec execute 333 sec latency 50.210 ms 2024-03-11T20:43:50.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:49 smithi204 ceph-mon[53271]: pgmap v84: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 791 KiB/s, 33 objects/s recovering 2024-03-11T20:43:51.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209897 10.88 MB/sec execute 334 sec latency 45.347 ms 2024-03-11T20:43:51.986 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:51 smithi083 ceph-mon[66365]: pgmap v85: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 691 KiB/s, 28 objects/s recovering 2024-03-11T20:43:52.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210198 10.86 MB/sec execute 335 sec latency 29.718 ms 2024-03-11T20:43:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:51 smithi204 ceph-mon[53271]: pgmap v85: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 691 KiB/s, 28 objects/s recovering 2024-03-11T20:43:53.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:52 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:53.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210497 10.85 MB/sec execute 336 sec latency 51.652 ms 2024-03-11T20:43:53.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:52 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:54.018 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:53 smithi083 ceph-mon[66365]: pgmap v86: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 572 op/s; 623 KiB/s, 26 objects/s recovering 2024-03-11T20:43:54.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210842 10.84 MB/sec execute 337 sec latency 65.055 ms 2024-03-11T20:43:54.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:53 smithi204 ceph-mon[53271]: pgmap v86: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 572 op/s; 623 KiB/s, 26 objects/s recovering 2024-03-11T20:43:55.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211291 10.84 MB/sec execute 338 sec latency 44.307 ms 2024-03-11T20:43:56.034 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211632 10.82 MB/sec execute 339 sec latency 53.117 ms 2024-03-11T20:43:56.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:55 smithi083 ceph-mon[66365]: pgmap v87: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s; 0 B/s, 9 objects/s recovering 2024-03-11T20:43:56.047 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:55 smithi204 ceph-mon[53271]: pgmap v87: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s; 0 B/s, 9 objects/s recovering 2024-03-11T20:43:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:57.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211875 10.81 MB/sec execute 340 sec latency 45.353 ms 2024-03-11T20:43:57.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:56 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:57.076 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:56 smithi083 ceph-mon[66365]: Upgrade: osd.1 is safe to restart 2024-03-11T20:43:57.076 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:57.076 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:57.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:56 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-11T20:43:57.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:56 smithi204 ceph-mon[53271]: Upgrade: osd.1 is safe to restart 2024-03-11T20:43:57.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:57.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:43:58.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 212181 10.82 MB/sec execute 341 sec latency 40.692 ms 2024-03-11T20:43:58.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:57 smithi083 ceph-mon[66365]: pgmap v88: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 0 B/s, 9 objects/s recovering 2024-03-11T20:43:58.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:58.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:43:58.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:58.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:57 smithi204 ceph-mon[53271]: pgmap v88: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 0 B/s, 9 objects/s recovering 2024-03-11T20:43:58.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:43:58.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-11T20:43:58.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:43:59.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 212535 10.83 MB/sec execute 342 sec latency 42.784 ms 2024-03-11T20:43:59.057 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:43:59.148 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:58 smithi083 ceph-mon[66365]: Upgrade: Updating osd.1 2024-03-11T20:43:59.148 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:58 smithi083 ceph-mon[66365]: Deploying daemon osd.1 on smithi083 2024-03-11T20:43:59.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:58 smithi204 ceph-mon[53271]: Upgrade: Updating osd.1 2024-03-11T20:43:59.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:58 smithi204 ceph-mon[53271]: Deploying daemon osd.1 on smithi083 2024-03-11T20:44:00.009 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:44:00.012 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 25s ago 14m 16.1M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (75s) 25s ago 14m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (68s) 63s ago 13m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (4m) 25s ago 14m 44.9M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (8m) 25s ago 8m 39.1M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (8m) 63s ago 8m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (3m) 25s ago 15m 500M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (3m) 63s ago 13m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (2m) 25s ago 15m 45.9M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (2m) 63s ago 13m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (7m) 25s ago 7m 289M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (7m) 63s ago 7m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (4m) 25s ago 13m 9.84M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (4m) 63s ago 13m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:44:00.013 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (35s) 25s ago 12m 11.8M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (12m) 25s ago 12m 356M 1936M 17.2.0 e1d6a67b021e abf1f4119cc1 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 25s ago 11m 492M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 25s ago 11m 362M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (10m) 63s ago 10m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (10m) 63s ago 10m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (10m) 63s ago 10m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (9m) 63s ago 9m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:44:00.014 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (3m) 25s ago 13m 67.6M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:44:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 212882 10.84 MB/sec execute 343 sec latency 34.388 ms 2024-03-11T20:44:00.160 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:59 smithi083 ceph-mon[66365]: pgmap v89: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 549 op/s 2024-03-11T20:44:00.161 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:43:59 smithi083 ceph-mon[66365]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:59 smithi204 ceph-mon[53271]: pgmap v89: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 549 op/s 2024-03-11T20:44:00.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:43:59 smithi204 ceph-mon[53271]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-11T20:44:00.601 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 5 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:44:00.602 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:44:01.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 213175 10.84 MB/sec execute 344 sec latency 43.138 ms 2024-03-11T20:44:01.075 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "7/23 daemons upgraded", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:44:01.076 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:44:01.077 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:44:01.174 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:00 smithi083 ceph-mon[66365]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:01.175 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:00 smithi083 ceph-mon[66365]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:01.175 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:00 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3750265692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:01.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:00 smithi204 ceph-mon[53271]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:00 smithi204 ceph-mon[53271]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:00 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3750265692' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:01.660 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:44:01.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:01 smithi083 ceph-mon[66365]: pgmap v90: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 399 op/s 2024-03-11T20:44:01.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:01 smithi083 ceph-mon[66365]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:02.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 213454 10.81 MB/sec execute 345 sec latency 42.075 ms 2024-03-11T20:44:02.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:01 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/4211278471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:44:02.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:01 smithi204 ceph-mon[53271]: pgmap v90: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 399 op/s 2024-03-11T20:44:02.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:01 smithi204 ceph-mon[53271]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:01 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/4211278471' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:44:03.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 213769 10.80 MB/sec execute 346 sec latency 51.131 ms 2024-03-11T20:44:03.461 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:03 smithi083 systemd[1]: Stopping Ceph osd.1 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:44:03.970 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:03 smithi083 ceph-mon[66365]: pgmap v91: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-03-11T20:44:03.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:03 smithi083 ceph-mon[66365]: osd.1 marked itself down and dead 2024-03-11T20:44:03.971 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:03 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[38025]: 2024-03-11T20:44:03.730+0000 7fa9123e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:44:03.971 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:03 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[38025]: 2024-03-11T20:44:03.730+0000 7fa9123e6700 -1 osd.1 83 *** Got signal Terminated *** 2024-03-11T20:44:03.971 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:03 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[38025]: 2024-03-11T20:44:03.730+0000 7fa9123e6700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:44:04.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214118 10.80 MB/sec execute 347 sec latency 92.180 ms 2024-03-11T20:44:04.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:03 smithi204 ceph-mon[53271]: pgmap v91: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.5 MiB/s wr, 563 op/s 2024-03-11T20:44:04.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:03 smithi204 ceph-mon[53271]: osd.1 marked itself down and dead 2024-03-11T20:44:04.834 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:04 smithi083 podman[74640]: 2024-03-11 20:44:04.603278743 +0000 UTC m=+1.018354144 container died abf1f4119cc1832556aa619532f28b426b897952493c1192e08cd144479f2d39 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , ceph=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, name=centos-stream, architecture=x86_64) 2024-03-11T20:44:05.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214211 10.78 MB/sec execute 348 sec latency 1031.072 ms 2024-03-11T20:44:05.087 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:04 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:44:05.088 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:04 smithi083 ceph-mon[66365]: osdmap e84: 8 total, 7 up, 8 in 2024-03-11T20:44:05.088 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:04 smithi083 podman[74640]: 2024-03-11 20:44:04.95601981 +0000 UTC m=+1.371095204 container cleanup abf1f4119cc1832556aa619532f28b426b897952493c1192e08cd144479f2d39 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, architecture=x86_64, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754) 2024-03-11T20:44:05.088 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:04 smithi083 bash[74640]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1 2024-03-11T20:44:05.350 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:05 smithi083 podman[74656]: 2024-03-11 20:44:05.208657624 +0000 UTC m=+0.601435710 container remove abf1f4119cc1832556aa619532f28b426b897952493c1192e08cd144479f2d39 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, vendor=Red Hat, Inc., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, release=754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_BRANCH=HEAD) 2024-03-11T20:44:05.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:04 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:44:05.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:04 smithi204 ceph-mon[53271]: osdmap e84: 8 total, 7 up, 8 in 2024-03-11T20:44:05.852 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:05 smithi083 podman[74742]: 2024-03-11 20:44:05.653756108 +0000 UTC m=+0.033920464 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:44:05.852 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:05 smithi083 podman[74742]: 2024-03-11 20:44:05.805524913 +0000 UTC m=+0.185689264 container create 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:44:06.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214755 10.77 MB/sec execute 349 sec latency 41.621 ms 2024-03-11T20:44:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:05 smithi083 ceph-mon[66365]: pgmap v93: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-11T20:44:06.103 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:05 smithi083 ceph-mon[66365]: osdmap e85: 8 total, 7 up, 8 in 2024-03-11T20:44:06.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:05 smithi204 ceph-mon[53271]: pgmap v93: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-11T20:44:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:05 smithi204 ceph-mon[53271]: osdmap e85: 8 total, 7 up, 8 in 2024-03-11T20:44:06.609 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:06 smithi083 podman[74742]: 2024-03-11 20:44:06.432236116 +0000 UTC m=+0.812400482 container init 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, GIT_CLEAN=True, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-11T20:44:06.609 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:06 smithi083 podman[74742]: 2024-03-11 20:44:06.443514043 +0000 UTC m=+0.823678393 container start 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-11T20:44:06.609 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:06 smithi083 podman[74742]: 2024-03-11 20:44:06.598564177 +0000 UTC m=+0.978728543 container attach 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-11T20:44:07.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 215255 10.76 MB/sec execute 350 sec latency 43.010 ms 2024-03-11T20:44:07.112 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:06 smithi083 podman[74742]: 2024-03-11 20:44:06.916960612 +0000 UTC m=+1.297124974 container died 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0) 2024-03-11T20:44:07.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:07 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 23/1686 objects degraded (1.364%), 2 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:07.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:07 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 23/1686 objects degraded (1.364%), 2 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:08.035 INFO:teuthology.orchestra.run.smithi083.stdout: 5 215765 10.79 MB/sec execute 351 sec latency 105.701 ms 2024-03-11T20:44:08.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:08 smithi083 ceph-mon[66365]: pgmap v95: 97 pgs: 2 active+undersized+degraded, 12 stale+active+clean, 2 active+undersized, 81 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.7 MiB/s wr, 307 op/s; 23/1686 objects degraded (1.364%) 2024-03-11T20:44:08.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:08 smithi204 ceph-mon[53271]: pgmap v95: 97 pgs: 2 active+undersized+degraded, 12 stale+active+clean, 2 active+undersized, 81 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.7 MiB/s wr, 307 op/s; 23/1686 objects degraded (1.364%) 2024-03-11T20:44:08.648 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:08 smithi083 podman[74766]: 2024-03-11 20:44:08.593514498 +0000 UTC m=+1.667608538 container remove 70c6c818f4b8373ae74e45006762820ea8aff0435a0d23b84923adfa52e0287f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307) 2024-03-11T20:44:08.648 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:08 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-11T20:44:08.648 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:08 smithi083 systemd[1]: Stopped Ceph osd.1 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:44:08.648 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:08 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.1.service: Consumed 3min 51.128s CPU time. 2024-03-11T20:44:09.037 INFO:teuthology.orchestra.run.smithi083.stdout: 5 216280 10.82 MB/sec execute 352 sec latency 35.918 ms 2024-03-11T20:44:09.173 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:08 smithi083 systemd[1]: Starting Ceph osd.1 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:44:09.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:09 smithi083 podman[74862]: 2024-03-11 20:44:09.182698057 +0000 UTC m=+0.039828754 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:44:09.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:09 smithi083 podman[74862]: 2024-03-11 20:44:09.335705075 +0000 UTC m=+0.192835760 container create a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:44:09.678 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:09 smithi083 podman[74862]: 2024-03-11 20:44:09.672586799 +0000 UTC m=+0.529717502 container init a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, ceph=True, RELEASE=main-2b0101e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:44:09.678 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:09 smithi083 podman[74862]: 2024-03-11 20:44:09.68139363 +0000 UTC m=+0.538524323 container start a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-03-11T20:44:09.929 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:09 smithi083 podman[74862]: 2024-03-11 20:44:09.831555441 +0000 UTC m=+0.688686129 container attach a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:44:10.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 216637 10.83 MB/sec execute 353 sec latency 31.577 ms 2024-03-11T20:44:10.185 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-mon[66365]: pgmap v96: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 481 op/s; 188/1704 objects degraded (11.033%) 2024-03-11T20:44:10.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:10 smithi204 ceph-mon[53271]: pgmap v96: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 481 op/s; 188/1704 objects degraded (11.033%) 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 bash[74862]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-11T20:44:10.436 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-11T20:44:10.437 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-11T20:44:10.437 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-11T20:44:10.437 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate[74877]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-11T20:44:10.437 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:10 smithi083 podman[74862]: 2024-03-11 20:44:10.369106709 +0000 UTC m=+1.226237395 container died a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-11T20:44:11.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 216892 10.81 MB/sec execute 354 sec latency 32.189 ms 2024-03-11T20:44:11.455 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 podman[75014]: 2024-03-11 20:44:11.30135732 +0000 UTC m=+0.925399286 container remove a75cb6cfe5a2ed89be4a72151ab4bd49c047708bc4eb4f5b63264df62cdb4fd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-11T20:44:11.718 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 podman[75074]: 2024-03-11 20:44:11.452393837 +0000 UTC m=+0.020209315 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:44:11.718 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 podman[75074]: 2024-03-11 20:44:11.590582809 +0000 UTC m=+0.158398288 container create ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-11T20:44:11.969 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 podman[75074]: 2024-03-11 20:44:11.868726096 +0000 UTC m=+0.436541582 container init ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, io.buildah.version=1.33.5) 2024-03-11T20:44:11.969 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 podman[75074]: 2024-03-11 20:44:11.875213978 +0000 UTC m=+0.443029473 container start ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD) 2024-03-11T20:44:11.969 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 ceph-osd[75093]: -- 172.21.15.83:0/2980073066 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5569c5527680 con 0x5569c54f1000 2024-03-11T20:44:12.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217213 10.78 MB/sec execute 355 sec latency 39.319 ms 2024-03-11T20:44:12.220 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 bash[75074]: ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 2024-03-11T20:44:12.220 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:11 smithi083 systemd[1]: Started Ceph osd.1 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:44:12.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:12 smithi083 ceph-mon[66365]: pgmap v97: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 404 op/s; 189/1707 objects degraded (11.072%) 2024-03-11T20:44:12.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:12.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:12.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:12 smithi204 ceph-mon[53271]: pgmap v97: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 404 op/s; 189/1707 objects degraded (11.072%) 2024-03-11T20:44:12.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:12.482 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:44:12.458+0000 7fe53f3d17c0 -1 Falling back to public interface 2024-03-11T20:44:13.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217441 10.77 MB/sec execute 356 sec latency 48.499 ms 2024-03-11T20:44:13.291 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:13.291 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:13.291 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:44:13.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:13.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:44:14.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217779 10.77 MB/sec execute 357 sec latency 49.660 ms 2024-03-11T20:44:14.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:14 smithi083 ceph-mon[66365]: pgmap v98: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.0 MiB/s wr, 603 op/s; 197/1746 objects degraded (11.283%) 2024-03-11T20:44:14.063 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:14 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 197/1746 objects degraded (11.283%), 24 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:14.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:14 smithi204 ceph-mon[53271]: pgmap v98: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.0 MiB/s wr, 603 op/s; 197/1746 objects degraded (11.283%) 2024-03-11T20:44:14.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:14 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 197/1746 objects degraded (11.283%), 24 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:15.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 218387 10.77 MB/sec execute 358 sec latency 34.519 ms 2024-03-11T20:44:16.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 218848 10.76 MB/sec execute 359 sec latency 45.213 ms 2024-03-11T20:44:16.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:16 smithi083 ceph-mon[66365]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 542 op/s; 196/1743 objects degraded (11.245%) 2024-03-11T20:44:16.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:16 smithi204 ceph-mon[53271]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 542 op/s; 196/1743 objects degraded (11.245%) 2024-03-11T20:44:17.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 219306 10.79 MB/sec execute 360 sec latency 53.488 ms 2024-03-11T20:44:18.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 219731 10.80 MB/sec execute 361 sec latency 39.421 ms 2024-03-11T20:44:18.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:18 smithi083 ceph-mon[66365]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 471 op/s; 192/1749 objects degraded (10.978%) 2024-03-11T20:44:18.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:18 smithi204 ceph-mon[53271]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 471 op/s; 192/1749 objects degraded (10.978%) 2024-03-11T20:44:19.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220065 10.81 MB/sec execute 362 sec latency 53.598 ms 2024-03-11T20:44:19.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:19 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 180/1683 objects degraded (10.695%), 24 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:19.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:19 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 180/1683 objects degraded (10.695%), 24 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:19.444 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:19 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:44:19.340+0000 7fe53f3d17c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-03-11T20:44:20.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220291 10.80 MB/sec execute 363 sec latency 41.355 ms 2024-03-11T20:44:20.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:20 smithi083 ceph-mon[66365]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 703 op/s; 180/1683 objects degraded (10.695%) 2024-03-11T20:44:20.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:20 smithi204 ceph-mon[53271]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 703 op/s; 180/1683 objects degraded (10.695%) 2024-03-11T20:44:20.706 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:20 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:44:20.640+0000 7fe53f3d17c0 -1 osd.1 83 log_to_monitors true 2024-03-11T20:44:21.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220574 10.77 MB/sec execute 364 sec latency 43.578 ms 2024-03-11T20:44:21.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:21 smithi083 ceph-mon[66365]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-11T20:44:21.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:21 smithi204 ceph-mon[53271]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-11T20:44:21.725 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:44:21 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:44:21.646+0000 7fe52c821700 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:44:22.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220770 10.75 MB/sec execute 365 sec latency 119.781 ms 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 581 op/s; 182/1683 objects degraded (10.814%) 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: osdmap e86: 8 total, 7 up, 8 in 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:22.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:22 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:22.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 581 op/s; 182/1683 objects degraded (10.814%) 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: osdmap e86: 8 total, 7 up, 8 in 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: from='osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859]' entity='osd.1' 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:22 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:23.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220929 10.73 MB/sec execute 366 sec latency 437.426 ms 2024-03-11T20:44:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:23 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:44:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:23 smithi204 ceph-mon[53271]: osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859] boot 2024-03-11T20:44:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:23 smithi204 ceph-mon[53271]: osdmap e87: 8 total, 8 up, 8 in 2024-03-11T20:44:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:44:23.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:23 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:44:23.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:23 smithi083 ceph-mon[66365]: osd.1 [v2:172.21.15.83:6810/1858841859,v1:172.21.15.83:6811/1858841859] boot 2024-03-11T20:44:23.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:23 smithi083 ceph-mon[66365]: osdmap e87: 8 total, 8 up, 8 in 2024-03-11T20:44:23.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-11T20:44:24.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220929 10.70 MB/sec execute 367 sec latency 1437.500 ms 2024-03-11T20:44:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:24 smithi083 ceph-mon[66365]: pgmap v105: 97 pgs: 20 peering, 12 active+undersized+degraded, 7 active+undersized, 58 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.5 MiB/s wr, 649 op/s; 80/1698 objects degraded (4.711%) 2024-03-11T20:44:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:24 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:24 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:24 smithi083 ceph-mon[66365]: osdmap e88: 8 total, 8 up, 8 in 2024-03-11T20:44:24.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:24 smithi204 ceph-mon[53271]: pgmap v105: 97 pgs: 20 peering, 12 active+undersized+degraded, 7 active+undersized, 58 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.5 MiB/s wr, 649 op/s; 80/1698 objects degraded (4.711%) 2024-03-11T20:44:24.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:24 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:24.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:24 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:24.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:24 smithi204 ceph-mon[53271]: osdmap e88: 8 total, 8 up, 8 in 2024-03-11T20:44:25.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220944 10.67 MB/sec execute 368 sec latency 2338.528 ms 2024-03-11T20:44:25.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:25 smithi083 ceph-mon[66365]: osdmap e89: 8 total, 8 up, 8 in 2024-03-11T20:44:25.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:25 smithi083 ceph-mon[66365]: pgmap v108: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 476 op/s; 70/1698 objects degraded (4.122%) 2024-03-11T20:44:25.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:25 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 80/1698 objects degraded (4.711%), 12 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:25.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:25 smithi083 ceph-mon[66365]: osdmap e90: 8 total, 8 up, 8 in 2024-03-11T20:44:25.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:25 smithi204 ceph-mon[53271]: osdmap e89: 8 total, 8 up, 8 in 2024-03-11T20:44:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:25 smithi204 ceph-mon[53271]: pgmap v108: 97 pgs: 23 peering, 10 active+undersized+degraded, 6 active+undersized, 58 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 476 op/s; 70/1698 objects degraded (4.122%) 2024-03-11T20:44:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:25 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 80/1698 objects degraded (4.711%), 12 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:25.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:25 smithi204 ceph-mon[53271]: osdmap e90: 8 total, 8 up, 8 in 2024-03-11T20:44:26.036 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221268 10.68 MB/sec execute 369 sec latency 432.296 ms 2024-03-11T20:44:27.037 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221635 10.66 MB/sec execute 370 sec latency 848.800 ms 2024-03-11T20:44:27.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:27 smithi083 ceph-mon[66365]: osdmap e91: 8 total, 8 up, 8 in 2024-03-11T20:44:27.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:27 smithi083 ceph-mon[66365]: pgmap v111: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 peering, 2 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 259 KiB/s wr, 27 op/s; 55/1641 objects degraded (3.352%); 22 B/s, 54 keys/s, 3 objects/s recovering 2024-03-11T20:44:27.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:27.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:27.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:27 smithi204 ceph-mon[53271]: osdmap e91: 8 total, 8 up, 8 in 2024-03-11T20:44:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:27 smithi204 ceph-mon[53271]: pgmap v111: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 24 peering, 2 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 259 KiB/s wr, 27 op/s; 55/1641 objects degraded (3.352%); 22 B/s, 54 keys/s, 3 objects/s recovering 2024-03-11T20:44:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:28.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221941 10.64 MB/sec execute 371 sec latency 434.151 ms 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: osdmap e92: 8 total, 8 up, 8 in 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:44:28.472 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:28 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: osdmap e92: 8 total, 8 up, 8 in 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:28.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:28 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:29.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222132 10.63 MB/sec execute 372 sec latency 860.934 ms 2024-03-11T20:44:29.491 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:29 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:29.491 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:29 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-03-11T20:44:29.491 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:29 smithi083 ceph-mon[66365]: osdmap e93: 8 total, 8 up, 8 in 2024-03-11T20:44:29.491 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:29 smithi083 ceph-mon[66365]: pgmap v114: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 2 active+undersized+degraded, 90 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 336 op/s; 55/1500 objects degraded (3.667%); 949 KiB/s, 54 keys/s, 46 objects/s recovering 2024-03-11T20:44:29.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:29 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:29.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:29 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-03-11T20:44:29.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:29 smithi204 ceph-mon[53271]: osdmap e93: 8 total, 8 up, 8 in 2024-03-11T20:44:29.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:29 smithi204 ceph-mon[53271]: pgmap v114: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 2 active+undersized+degraded, 90 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 336 op/s; 55/1500 objects degraded (3.667%); 949 KiB/s, 54 keys/s, 46 objects/s recovering 2024-03-11T20:44:30.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222467 10.62 MB/sec execute 373 sec latency 40.582 ms 2024-03-11T20:44:30.512 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:30 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 55/1500 objects degraded (3.667%), 3 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:30.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:30 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 55/1500 objects degraded (3.667%), 3 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:31.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222781 10.63 MB/sec execute 374 sec latency 47.724 ms 2024-03-11T20:44:31.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:31 smithi083 ceph-mon[66365]: pgmap v115: 97 pgs: 4 peering, 2 active+undersized+degraded, 91 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 MiB/s wr, 273 op/s; 26/1551 objects degraded (1.676%); 18 KiB/s, 40 keys/s, 39 objects/s recovering 2024-03-11T20:44:31.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:31 smithi204 ceph-mon[53271]: pgmap v115: 97 pgs: 4 peering, 2 active+undersized+degraded, 91 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 MiB/s wr, 273 op/s; 26/1551 objects degraded (1.676%); 18 KiB/s, 40 keys/s, 39 objects/s recovering 2024-03-11T20:44:32.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223062 10.62 MB/sec execute 375 sec latency 56.715 ms 2024-03-11T20:44:32.071 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:44:32.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:32 smithi083 ceph-mon[66365]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:32.842 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 10s ago 14m 16.1M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (108s) 10s ago 14m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (101s) 95s ago 14m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 10s ago 14m 45.0M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (8m) 10s ago 8m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (8m) 95s ago 8m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (4m) 10s ago 16m 505M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (3m) 95s ago 14m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (3m) 10s ago 16m 48.2M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (2m) 95s ago 14m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (8m) 10s ago 8m 286M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (8m) 95s ago 8m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 10s ago 14m 11.4M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:44:32.843 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (4m) 95s ago 13m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (68s) 10s ago 13m 250M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (21s) 10s ago 12m 11.5M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (12m) 10s ago 12m 532M 1936M 17.2.0 e1d6a67b021e 92664e8f449d 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 10s ago 11m 408M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (11m) 95s ago 11m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (11m) 95s ago 11m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (10m) 95s ago 10m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (10m) 95s ago 10m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:44:32.844 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (3m) 10s ago 13m 69.9M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:44:32.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:32 smithi204 ceph-mon[53271]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:33.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223344 10.63 MB/sec execute 376 sec latency 66.388 ms 2024-03-11T20:44:33.328 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:44:33.328 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:44:33.328 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:44:33.328 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 6 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:44:33.329 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:44:33.695 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:33 smithi083 ceph-mon[66365]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:33.695 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:33 smithi083 ceph-mon[66365]: pgmap v116: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.9 MiB/s wr, 513 op/s; 1.3 MiB/s, 40 objects/s recovering 2024-03-11T20:44:33.695 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:33 smithi083 ceph-mon[66365]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:33.695 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:33 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1655092808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:44:33.726 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "8/23 daemons upgraded", 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:44:33.727 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:44:33.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:33 smithi204 ceph-mon[53271]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:33.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:33 smithi204 ceph-mon[53271]: pgmap v116: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.9 MiB/s wr, 513 op/s; 1.3 MiB/s, 40 objects/s recovering 2024-03-11T20:44:33.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:33 smithi204 ceph-mon[53271]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:33.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:33 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1655092808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:44:34.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223574 10.63 MB/sec execute 377 sec latency 55.496 ms 2024-03-11T20:44:34.190 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:44:34.708 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:34 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1551 objects degraded (1.676%), 2 pgs degraded) 2024-03-11T20:44:34.708 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:34 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:44:34.708 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:34 smithi083 ceph-mon[66365]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:34.708 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:34 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1140726898' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:44:34.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:34 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1551 objects degraded (1.676%), 2 pgs degraded) 2024-03-11T20:44:34.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:34 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:44:34.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:34 smithi204 ceph-mon[53271]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:44:34.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:34 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1140726898' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:44:35.043 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223854 10.62 MB/sec execute 378 sec latency 40.401 ms 2024-03-11T20:44:35.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:35 smithi083 ceph-mon[66365]: pgmap v117: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 411 op/s; 1.0 MiB/s, 32 objects/s recovering 2024-03-11T20:44:35.928 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:35 smithi204 ceph-mon[53271]: pgmap v117: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 411 op/s; 1.0 MiB/s, 32 objects/s recovering 2024-03-11T20:44:36.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 224266 10.60 MB/sec execute 379 sec latency 27.257 ms 2024-03-11T20:44:37.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 224639 10.60 MB/sec execute 380 sec latency 45.027 ms 2024-03-11T20:44:37.921 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:37 smithi083 ceph-mon[66365]: pgmap v118: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 885 KiB/s, 27 objects/s recovering 2024-03-11T20:44:38.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 225173 10.60 MB/sec execute 381 sec latency 45.435 ms 2024-03-11T20:44:38.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:37 smithi204 ceph-mon[53271]: pgmap v118: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 885 KiB/s, 27 objects/s recovering 2024-03-11T20:44:39.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 225689 10.60 MB/sec execute 382 sec latency 49.373 ms 2024-03-11T20:44:39.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:39 smithi083 ceph-mon[66365]: pgmap v119: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 434 KiB/s, 8 objects/s recovering 2024-03-11T20:44:40.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 226130 10.61 MB/sec execute 383 sec latency 48.913 ms 2024-03-11T20:44:40.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:39 smithi204 ceph-mon[53271]: pgmap v119: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 434 KiB/s, 8 objects/s recovering 2024-03-11T20:44:41.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 226582 10.62 MB/sec execute 384 sec latency 29.995 ms 2024-03-11T20:44:41.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:41 smithi083 ceph-mon[66365]: pgmap v120: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 374 KiB/s, 7 objects/s recovering 2024-03-11T20:44:42.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227027 10.65 MB/sec execute 385 sec latency 39.945 ms 2024-03-11T20:44:42.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:41 smithi204 ceph-mon[53271]: pgmap v120: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 374 KiB/s, 7 objects/s recovering 2024-03-11T20:44:42.991 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:42.991 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:42.991 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:43.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227256 10.64 MB/sec execute 386 sec latency 43.756 ms 2024-03-11T20:44:43.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:43.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:43.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:44.022 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:43 smithi083 ceph-mon[66365]: pgmap v121: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.4 MiB/s wr, 594 op/s; 341 KiB/s, 4 objects/s recovering 2024-03-11T20:44:44.022 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:44.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227460 10.62 MB/sec execute 387 sec latency 76.914 ms 2024-03-11T20:44:44.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:43 smithi204 ceph-mon[53271]: pgmap v121: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.4 MiB/s wr, 594 op/s; 341 KiB/s, 4 objects/s recovering 2024-03-11T20:44:44.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:43 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:45.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:44 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:45.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:44 smithi083 ceph-mon[66365]: Upgrade: osd.2 is safe to restart 2024-03-11T20:44:45.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227650 10.60 MB/sec execute 388 sec latency 73.382 ms 2024-03-11T20:44:45.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:44 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-11T20:44:45.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:44 smithi204 ceph-mon[53271]: Upgrade: osd.2 is safe to restart 2024-03-11T20:44:46.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227850 10.58 MB/sec execute 389 sec latency 89.245 ms 2024-03-11T20:44:46.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:45 smithi083 ceph-mon[66365]: pgmap v122: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-03-11T20:44:46.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:45 smithi204 ceph-mon[53271]: pgmap v122: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-03-11T20:44:47.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228059 10.56 MB/sec execute 390 sec latency 91.561 ms 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: Upgrade: Updating osd.2 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: Deploying daemon osd.2 on smithi083 2024-03-11T20:44:47.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:47 smithi083 ceph-mon[66365]: pgmap v123: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 497 op/s 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: Upgrade: Updating osd.2 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: Deploying daemon osd.2 on smithi083 2024-03-11T20:44:47.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:47 smithi204 ceph-mon[53271]: pgmap v123: 97 pgs: 97 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 497 op/s 2024-03-11T20:44:48.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228385 10.56 MB/sec execute 391 sec latency 53.697 ms 2024-03-11T20:44:49.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228778 10.55 MB/sec execute 392 sec latency 46.270 ms 2024-03-11T20:44:49.981 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:49 smithi083 ceph-mon[66365]: pgmap v124: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-11T20:44:50.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229127 10.54 MB/sec execute 393 sec latency 51.578 ms 2024-03-11T20:44:50.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:49 smithi204 ceph-mon[53271]: pgmap v124: 97 pgs: 97 active+clean; 329 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-03-11T20:44:51.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229368 10.53 MB/sec execute 394 sec latency 58.801 ms 2024-03-11T20:44:52.022 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:51 smithi083 ceph-mon[66365]: pgmap v125: 97 pgs: 97 active+clean; 327 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-03-11T20:44:52.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229732 10.53 MB/sec execute 395 sec latency 75.105 ms 2024-03-11T20:44:52.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:51 smithi204 ceph-mon[53271]: pgmap v125: 97 pgs: 97 active+clean; 327 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-03-11T20:44:52.290 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:52 smithi083 systemd[1]: Stopping Ceph osd.2 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:44:52.560 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:52 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[41881]: 2024-03-11T20:44:52.542+0000 7f2be1cec700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:44:52.560 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:52 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[41881]: 2024-03-11T20:44:52.542+0000 7f2be1cec700 -1 osd.2 93 *** Got signal Terminated *** 2024-03-11T20:44:52.560 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:52 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[41881]: 2024-03-11T20:44:52.542+0000 7f2be1cec700 -1 osd.2 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:44:53.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229962 10.53 MB/sec execute 396 sec latency 191.383 ms 2024-03-11T20:44:53.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:52 smithi204 ceph-mon[53271]: osd.2 marked itself down and dead 2024-03-11T20:44:53.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:52 smithi083 ceph-mon[66365]: osd.2 marked itself down and dead 2024-03-11T20:44:53.770 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:53 smithi083 podman[77217]: 2024-03-11 20:44:53.520541024 +0000 UTC m=+1.114035532 container died 92664e8f449d233b16dac04ff1fb8e03445447ef39919c5945a3eca1f4883cb1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, ceph=True, RELEASE=HEAD) 2024-03-11T20:44:54.021 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:53 smithi083 ceph-mon[66365]: pgmap v126: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 554 op/s 2024-03-11T20:44:54.021 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:53 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:44:54.021 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:53 smithi083 ceph-mon[66365]: osdmap e94: 8 total, 7 up, 8 in 2024-03-11T20:44:54.049 INFO:teuthology.orchestra.run.smithi083.stdout: 5 230023 10.51 MB/sec execute 397 sec latency 1051.522 ms 2024-03-11T20:44:54.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:53 smithi204 ceph-mon[53271]: pgmap v126: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 554 op/s 2024-03-11T20:44:54.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:53 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:44:54.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:53 smithi204 ceph-mon[53271]: osdmap e94: 8 total, 7 up, 8 in 2024-03-11T20:44:54.287 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:54 smithi083 podman[77217]: 2024-03-11 20:44:54.1595175 +0000 UTC m=+1.753012009 container cleanup 92664e8f449d233b16dac04ff1fb8e03445447ef39919c5945a3eca1f4883cb1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, name=centos-stream, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, ceph=True, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8) 2024-03-11T20:44:54.287 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:54 smithi083 bash[77217]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2 2024-03-11T20:44:54.564 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:54 smithi083 podman[77232]: 2024-03-11 20:44:54.495743658 +0000 UTC m=+0.970741211 container remove 92664e8f449d233b16dac04ff1fb8e03445447ef39919c5945a3eca1f4883cb1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, io.openshift.expose-services=, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream) 2024-03-11T20:44:55.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 230369 10.52 MB/sec execute 398 sec latency 62.153 ms 2024-03-11T20:44:55.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:54 smithi083 ceph-mon[66365]: osdmap e95: 8 total, 7 up, 8 in 2024-03-11T20:44:55.077 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:54.94369584 +0000 UTC m=+0.025228028 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:44:55.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:54 smithi204 ceph-mon[53271]: osdmap e95: 8 total, 7 up, 8 in 2024-03-11T20:44:55.328 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:55.102024876 +0000 UTC m=+0.183557115 container create 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-11T20:44:55.579 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:55.451736438 +0000 UTC m=+0.533268630 container init 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:44:55.579 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:55.464159926 +0000 UTC m=+0.545692104 container start 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307) 2024-03-11T20:44:55.850 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:55.669136955 +0000 UTC m=+0.750669144 container attach 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-11T20:44:56.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 230677 10.52 MB/sec execute 399 sec latency 59.233 ms 2024-03-11T20:44:56.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:55 smithi083 ceph-mon[66365]: pgmap v129: 97 pgs: 7 stale+active+clean, 90 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 503 op/s 2024-03-11T20:44:56.105 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:55 smithi083 podman[77321]: 2024-03-11 20:44:55.899817039 +0000 UTC m=+0.981349214 container died 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD) 2024-03-11T20:44:56.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:55 smithi204 ceph-mon[53271]: pgmap v129: 97 pgs: 7 stale+active+clean, 90 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 503 op/s 2024-03-11T20:44:57.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 230998 10.52 MB/sec execute 400 sec latency 42.045 ms 2024-03-11T20:44:57.630 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:57 smithi083 podman[77345]: 2024-03-11 20:44:57.421478436 +0000 UTC m=+1.511422267 container remove 4647261536f7c0b0fbb43fc976b7748e934116b962810b6d201972ced4f526dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-11T20:44:57.630 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:57 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-11T20:44:57.630 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:57 smithi083 systemd[1]: Stopped Ceph osd.2 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:44:57.631 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:57 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.2.service: Consumed 2min 57.512s CPU time. 2024-03-11T20:44:57.887 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:57 smithi083 systemd[1]: Starting Ceph osd.2 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:44:57.887 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:57 smithi083 ceph-mon[66365]: pgmap v130: 97 pgs: 5 active+undersized+degraded, 4 stale+active+clean, 4 active+undersized, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 53/1626 objects degraded (3.260%) 2024-03-11T20:44:57.888 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:57.888 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:57.888 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:57 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 53/1626 objects degraded (3.260%), 5 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:58.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231322 10.50 MB/sec execute 401 sec latency 40.592 ms 2024-03-11T20:44:58.138 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:58 smithi083 podman[77440]: 2024-03-11 20:44:57.990971599 +0000 UTC m=+0.027000960 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:44:58.138 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:58 smithi083 podman[77440]: 2024-03-11 20:44:58.122807591 +0000 UTC m=+0.158836970 container create 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-11T20:44:58.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:57 smithi204 ceph-mon[53271]: pgmap v130: 97 pgs: 5 active+undersized+degraded, 4 stale+active+clean, 4 active+undersized, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 53/1626 objects degraded (3.260%) 2024-03-11T20:44:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:44:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:44:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:57 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 53/1626 objects degraded (3.260%), 5 pgs degraded (PG_DEGRADED) 2024-03-11T20:44:58.652 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:58 smithi083 podman[77440]: 2024-03-11 20:44:58.444073715 +0000 UTC m=+0.480103065 container init 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240307) 2024-03-11T20:44:58.652 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:58 smithi083 podman[77440]: 2024-03-11 20:44:58.452785408 +0000 UTC m=+0.488814753 container start 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e) 2024-03-11T20:44:58.653 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:58 smithi083 podman[77440]: 2024-03-11 20:44:58.628827634 +0000 UTC m=+0.664856977 container attach 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=) 2024-03-11T20:44:59.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231548 10.48 MB/sec execute 402 sec latency 39.754 ms 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 bash[77440]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-11T20:44:59.182 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-11T20:44:59.183 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-11T20:44:59.183 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-11T20:44:59.183 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate[77453]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-11T20:44:59.183 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:44:59 smithi083 podman[77440]: 2024-03-11 20:44:59.130394853 +0000 UTC m=+1.166424193 container died 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-11T20:44:59.936 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:44:59 smithi083 ceph-mon[66365]: pgmap v131: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 152/1695 objects degraded (8.968%) 2024-03-11T20:45:00.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231882 10.49 MB/sec execute 403 sec latency 53.718 ms 2024-03-11T20:45:00.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:44:59 smithi204 ceph-mon[53271]: pgmap v131: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 152/1695 objects degraded (8.968%) 2024-03-11T20:45:00.690 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:00 smithi083 podman[77579]: 2024-03-11 20:45:00.444204465 +0000 UTC m=+1.304744913 container remove 8705bac79c3c235e404cbc756bb754a8e9b2e2a5bbb0220d97a6049b66bde902 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-03-11T20:45:00.941 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:00 smithi083 podman[77635]: 2024-03-11 20:45:00.624016663 +0000 UTC m=+0.026597444 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:45:00.941 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:00 smithi083 podman[77635]: 2024-03-11 20:45:00.767487785 +0000 UTC m=+0.170068562 container create ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0) 2024-03-11T20:45:01.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 232245 10.47 MB/sec execute 404 sec latency 65.485 ms 2024-03-11T20:45:01.196 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:01 smithi083 podman[77635]: 2024-03-11 20:45:01.169457932 +0000 UTC m=+0.572038721 container init ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-11T20:45:01.196 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:01 smithi083 podman[77635]: 2024-03-11 20:45:01.174774084 +0000 UTC m=+0.577354867 container start ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:45:01.463 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:01 smithi083 bash[77635]: ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 2024-03-11T20:45:01.463 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:01 smithi083 systemd[1]: Started Ceph osd.2 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:45:01.978 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:01 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:45:01.761+0000 7f10abe297c0 -1 Falling back to public interface 2024-03-11T20:45:01.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:01 smithi083 ceph-mon[66365]: pgmap v132: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 150/1686 objects degraded (8.897%) 2024-03-11T20:45:01.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:01.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:01.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:02.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 232670 10.46 MB/sec execute 405 sec latency 87.324 ms 2024-03-11T20:45:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:01 smithi204 ceph-mon[53271]: pgmap v132: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 150/1686 objects degraded (8.897%) 2024-03-11T20:45:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:03.004 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:02 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 152/1599 objects degraded (9.506%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:03.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233003 10.46 MB/sec execute 406 sec latency 39.716 ms 2024-03-11T20:45:03.178 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:02 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 152/1599 objects degraded (9.506%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:04.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:03 smithi083 ceph-mon[66365]: pgmap v133: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 433 op/s; 152/1599 objects degraded (9.506%) 2024-03-11T20:45:04.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233458 10.48 MB/sec execute 407 sec latency 38.823 ms 2024-03-11T20:45:04.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:03 smithi204 ceph-mon[53271]: pgmap v133: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 433 op/s; 152/1599 objects degraded (9.506%) 2024-03-11T20:45:04.669 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:45:05.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233849 10.49 MB/sec execute 408 sec latency 33.321 ms 2024-03-11T20:45:05.692 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:45:05.692 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 43s ago 15m 16.1M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:45:05.692 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (2m) 43s ago 15m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:45:05.692 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (2m) 2m ago 14m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:45:05.692 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 43s ago 15m 45.0M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (9m) 43s ago 9m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (9m) 2m ago 9m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (5m) 43s ago 16m 505M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (4m) 2m ago 14m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (3m) 43s ago 16m 48.2M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (3m) 2m ago 14m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (9m) 43s ago 9m 286M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (8m) 2m ago 8m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 43s ago 15m 11.4M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (5m) 2m ago 14m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (101s) 43s ago 13m 250M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (54s) 43s ago 13m 11.5M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 starting - - - 1936M 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12m) 43s ago 12m 408M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (12m) 2m ago 12m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:45:05.693 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (11m) 2m ago 11m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:45:05.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (11m) 2m ago 11m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:45:05.694 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (10m) 2m ago 10m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:45:05.694 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 43s ago 14m 69.9M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:45:06.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:05 smithi083 ceph-mon[66365]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:06.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:05 smithi083 ceph-mon[66365]: pgmap v134: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 393 op/s; 152/1599 objects degraded (9.506%) 2024-03-11T20:45:06.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:05 smithi083 ceph-mon[66365]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:06.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234164 10.50 MB/sec execute 409 sec latency 34.620 ms 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:45:06.277 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 6 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:45:06.278 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:45:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:05 smithi204 ceph-mon[53271]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:05 smithi204 ceph-mon[53271]: pgmap v134: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 393 op/s; 152/1599 objects degraded (9.506%) 2024-03-11T20:45:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:05 smithi204 ceph-mon[53271]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:45:06.743 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "8/23 daemons upgraded", 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:45:06.744 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:45:07.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234385 10.48 MB/sec execute 410 sec latency 86.275 ms 2024-03-11T20:45:07.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:06 smithi083 ceph-mon[66365]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:07.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:06 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3401256632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 152/1629 objects degraded (9.331%), 19 pgs degraded 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: osd.2 (root=default,host=smithi083) is down 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 152/1629 objects degraded (9.331%), 19 pgs degraded 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-03-11T20:45:07.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-03-11T20:45:07.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-03-11T20:45:07.308 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-03-11T20:45:07.308 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-03-11T20:45:07.428 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:06 smithi204 ceph-mon[53271]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:06 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3401256632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:08.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234703 10.47 MB/sec execute 411 sec latency 42.977 ms 2024-03-11T20:45:08.104 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:07 smithi083 ceph-mon[66365]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:08.104 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:07 smithi083 ceph-mon[66365]: pgmap v135: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 332 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 152/1629 objects degraded (9.331%) 2024-03-11T20:45:08.104 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:08.104 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:07 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/4275632470' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:45:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:07 smithi204 ceph-mon[53271]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:07 smithi204 ceph-mon[53271]: pgmap v135: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 332 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 152/1629 objects degraded (9.331%) 2024-03-11T20:45:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:07 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/4275632470' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:45:08.613 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:08 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:45:08.414+0000 7f10abe297c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-03-11T20:45:09.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235047 10.46 MB/sec execute 412 sec latency 44.632 ms 2024-03-11T20:45:09.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:09 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 149/1674 objects degraded (8.901%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:09.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:09 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 149/1674 objects degraded (8.901%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:09.625 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:45:09.535+0000 7f10abe297c0 -1 osd.2 93 log_to_monitors true 2024-03-11T20:45:10.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235460 10.46 MB/sec execute 413 sec latency 37.781 ms 2024-03-11T20:45:10.134 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:10 smithi083 ceph-mon[66365]: pgmap v136: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 535 op/s; 149/1674 objects degraded (8.901%) 2024-03-11T20:45:10.134 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:10 smithi083 ceph-mon[66365]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-11T20:45:10.134 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:45:10 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:45:10.043+0000 7f109d281700 -1 osd.2 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:45:10.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:10 smithi204 ceph-mon[53271]: pgmap v136: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 535 op/s; 149/1674 objects degraded (8.901%) 2024-03-11T20:45:10.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:10 smithi204 ceph-mon[53271]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-11T20:45:11.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235921 10.45 MB/sec execute 414 sec latency 56.183 ms 2024-03-11T20:45:11.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:11 smithi204 ceph-mon[53271]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-11T20:45:11.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:11 smithi204 ceph-mon[53271]: osdmap e96: 8 total, 7 up, 8 in 2024-03-11T20:45:11.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:11 smithi204 ceph-mon[53271]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:45:11.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:11 smithi083 ceph-mon[66365]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-11T20:45:11.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:11 smithi083 ceph-mon[66365]: osdmap e96: 8 total, 7 up, 8 in 2024-03-11T20:45:11.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:11 smithi083 ceph-mon[66365]: from='osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:45:12.050 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235921 10.43 MB/sec execute 415 sec latency 1056.223 ms 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: pgmap v138: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 555 op/s; 152/1686 objects degraded (9.015%) 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401] boot 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: osdmap e97: 8 total, 8 up, 8 in 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: pgmap v138: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 555 op/s; 152/1686 objects degraded (9.015%) 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: osd.2 [v2:172.21.15.83:6818/3641314401,v1:172.21.15.83:6819/3641314401] boot 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: osdmap e97: 8 total, 8 up, 8 in 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:12.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:13.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235921 10.40 MB/sec execute 416 sec latency 2056.266 ms 2024-03-11T20:45:13.272 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:13 smithi083 ceph-mon[66365]: osdmap e98: 8 total, 8 up, 8 in 2024-03-11T20:45:13.272 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:13.272 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:13.272 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:13 smithi083 ceph-mon[66365]: osdmap e99: 8 total, 8 up, 8 in 2024-03-11T20:45:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:13 smithi204 ceph-mon[53271]: osdmap e98: 8 total, 8 up, 8 in 2024-03-11T20:45:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:13 smithi204 ceph-mon[53271]: osdmap e99: 8 total, 8 up, 8 in 2024-03-11T20:45:14.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236252 10.39 MB/sec execute 417 sec latency 2092.159 ms 2024-03-11T20:45:14.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:14 smithi083 ceph-mon[66365]: pgmap v141: 97 pgs: 7 remapped+peering, 4 peering, 5 active+undersized+degraded, 3 active+undersized, 78 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 6.6 MiB/s wr, 705 op/s; 31/1566 objects degraded (1.980%); 0 B/s, 2 objects/s recovering 2024-03-11T20:45:14.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:14 smithi204 ceph-mon[53271]: pgmap v141: 97 pgs: 7 remapped+peering, 4 peering, 5 active+undersized+degraded, 3 active+undersized, 78 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 6.6 MiB/s wr, 705 op/s; 31/1566 objects degraded (1.980%); 0 B/s, 2 objects/s recovering 2024-03-11T20:45:15.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236252 10.37 MB/sec execute 418 sec latency 1046.070 ms 2024-03-11T20:45:15.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:15 smithi083 ceph-mon[66365]: osdmap e100: 8 total, 8 up, 8 in 2024-03-11T20:45:15.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:15 smithi204 ceph-mon[53271]: osdmap e100: 8 total, 8 up, 8 in 2024-03-11T20:45:16.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236596 10.37 MB/sec execute 419 sec latency 1051.154 ms 2024-03-11T20:45:16.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:16 smithi083 ceph-mon[66365]: pgmap v144: 97 pgs: 7 remapped+peering, 4 peering, 5 active+undersized+degraded, 3 active+undersized, 78 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.6 MiB/s wr, 505 op/s; 31/1566 objects degraded (1.980%); 0 B/s, 3 objects/s recovering 2024-03-11T20:45:16.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:16 smithi083 ceph-mon[66365]: osdmap e101: 8 total, 8 up, 8 in 2024-03-11T20:45:16.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:16 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 31/1566 objects degraded (1.980%), 5 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:16.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:16 smithi083 ceph-mon[66365]: osdmap e102: 8 total, 8 up, 8 in 2024-03-11T20:45:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:16 smithi204 ceph-mon[53271]: pgmap v144: 97 pgs: 7 remapped+peering, 4 peering, 5 active+undersized+degraded, 3 active+undersized, 78 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.6 MiB/s wr, 505 op/s; 31/1566 objects degraded (1.980%); 0 B/s, 3 objects/s recovering 2024-03-11T20:45:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:16 smithi204 ceph-mon[53271]: osdmap e101: 8 total, 8 up, 8 in 2024-03-11T20:45:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:16 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 31/1566 objects degraded (1.980%), 5 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:16 smithi204 ceph-mon[53271]: osdmap e102: 8 total, 8 up, 8 in 2024-03-11T20:45:17.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237038 10.38 MB/sec execute 420 sec latency 533.357 ms 2024-03-11T20:45:17.663 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:17 smithi083 ceph-mon[66365]: osdmap e103: 8 total, 8 up, 8 in 2024-03-11T20:45:17.663 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:17 smithi083 ceph-mon[66365]: pgmap v148: 97 pgs: 6 remapped+peering, 3 peering, 88 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 907 KiB/s wr, 121 op/s; 800 KiB/s, 19 objects/s recovering 2024-03-11T20:45:17.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:17 smithi204 ceph-mon[53271]: osdmap e103: 8 total, 8 up, 8 in 2024-03-11T20:45:17.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:17 smithi204 ceph-mon[53271]: pgmap v148: 97 pgs: 6 remapped+peering, 3 peering, 88 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 907 KiB/s wr, 121 op/s; 800 KiB/s, 19 objects/s recovering 2024-03-11T20:45:18.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237402 10.39 MB/sec execute 421 sec latency 954.335 ms 2024-03-11T20:45:18.670 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:18 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1566 objects degraded (1.980%), 5 pgs degraded) 2024-03-11T20:45:18.670 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:18 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:45:18.670 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:18 smithi083 ceph-mon[66365]: osdmap e104: 8 total, 8 up, 8 in 2024-03-11T20:45:18.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:18 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1566 objects degraded (1.980%), 5 pgs degraded) 2024-03-11T20:45:18.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:18 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:45:18.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:18 smithi204 ceph-mon[53271]: osdmap e104: 8 total, 8 up, 8 in 2024-03-11T20:45:19.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237777 10.40 MB/sec execute 422 sec latency 46.331 ms 2024-03-11T20:45:19.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:19.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: pgmap v150: 97 pgs: 3 peering, 94 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 242 op/s; 2.7 MiB/s, 42 objects/s recovering 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:19.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:19 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-11T20:45:20.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238083 10.39 MB/sec execute 423 sec latency 52.346 ms 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: pgmap v150: 97 pgs: 3 peering, 94 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 242 op/s; 2.7 MiB/s, 42 objects/s recovering 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:19 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-11T20:45:21.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238395 10.38 MB/sec execute 424 sec latency 53.601 ms 2024-03-11T20:45:22.004 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:21 smithi083 ceph-mon[66365]: pgmap v151: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.9 MiB/s wr, 270 op/s; 1.4 MiB/s, 28 objects/s recovering 2024-03-11T20:45:22.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238660 10.37 MB/sec execute 425 sec latency 53.669 ms 2024-03-11T20:45:22.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:21 smithi204 ceph-mon[53271]: pgmap v151: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.9 MiB/s wr, 270 op/s; 1.4 MiB/s, 28 objects/s recovering 2024-03-11T20:45:23.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238991 10.37 MB/sec execute 426 sec latency 77.470 ms 2024-03-11T20:45:24.026 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:23 smithi083 ceph-mon[66365]: pgmap v152: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 6.0 MiB/s wr, 543 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-03-11T20:45:24.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 239363 10.36 MB/sec execute 427 sec latency 74.966 ms 2024-03-11T20:45:24.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:23 smithi204 ceph-mon[53271]: pgmap v152: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 6.0 MiB/s wr, 543 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-03-11T20:45:25.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 239701 10.34 MB/sec execute 428 sec latency 50.179 ms 2024-03-11T20:45:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240024 10.34 MB/sec execute 429 sec latency 40.803 ms 2024-03-11T20:45:26.082 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:25 smithi083 ceph-mon[66365]: pgmap v153: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 5.3 MiB/s wr, 475 op/s; 979 KiB/s, 11 objects/s recovering 2024-03-11T20:45:26.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:25 smithi204 ceph-mon[53271]: pgmap v153: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 5.3 MiB/s wr, 475 op/s; 979 KiB/s, 11 objects/s recovering 2024-03-11T20:45:27.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240389 10.35 MB/sec execute 430 sec latency 86.514 ms 2024-03-11T20:45:27.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:27 smithi083 ceph-mon[66365]: pgmap v154: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 517 op/s; 819 KiB/s, 9 objects/s recovering 2024-03-11T20:45:27.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:27.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:28.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240752 10.35 MB/sec execute 431 sec latency 51.661 ms 2024-03-11T20:45:28.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:27 smithi204 ceph-mon[53271]: pgmap v154: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 517 op/s; 819 KiB/s, 9 objects/s recovering 2024-03-11T20:45:28.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:28.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:29.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240991 10.36 MB/sec execute 432 sec latency 60.027 ms 2024-03-11T20:45:30.015 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:29 smithi083 ceph-mon[66365]: pgmap v155: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 721 KiB/s, 8 objects/s recovering 2024-03-11T20:45:30.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241194 10.35 MB/sec execute 433 sec latency 78.235 ms 2024-03-11T20:45:30.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:29 smithi204 ceph-mon[53271]: pgmap v155: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 721 KiB/s, 8 objects/s recovering 2024-03-11T20:45:31.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241400 10.34 MB/sec execute 434 sec latency 74.243 ms 2024-03-11T20:45:32.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241663 10.32 MB/sec execute 435 sec latency 108.458 ms 2024-03-11T20:45:32.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:31 smithi083 ceph-mon[66365]: pgmap v156: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 477 op/s 2024-03-11T20:45:32.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:31 smithi204 ceph-mon[53271]: pgmap v156: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 477 op/s 2024-03-11T20:45:33.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241947 10.32 MB/sec execute 436 sec latency 58.950 ms 2024-03-11T20:45:33.092 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:33.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:34.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242192 10.31 MB/sec execute 437 sec latency 54.934 ms 2024-03-11T20:45:34.125 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:33 smithi083 ceph-mon[66365]: pgmap v157: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 531 op/s 2024-03-11T20:45:34.125 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:33 smithi204 ceph-mon[53271]: pgmap v157: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 531 op/s 2024-03-11T20:45:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:35.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242559 10.30 MB/sec execute 438 sec latency 78.631 ms 2024-03-11T20:45:35.133 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:34 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:35.133 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:34 smithi083 ceph-mon[66365]: Upgrade: osd.3 is safe to restart 2024-03-11T20:45:35.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:34 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-11T20:45:35.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:34 smithi204 ceph-mon[53271]: Upgrade: osd.3 is safe to restart 2024-03-11T20:45:36.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243062 10.29 MB/sec execute 439 sec latency 41.615 ms 2024-03-11T20:45:36.147 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:35 smithi083 ceph-mon[66365]: pgmap v158: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.5 MiB/s wr, 325 op/s 2024-03-11T20:45:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:35 smithi204 ceph-mon[53271]: pgmap v158: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.5 MiB/s wr, 325 op/s 2024-03-11T20:45:37.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243482 10.29 MB/sec execute 440 sec latency 34.832 ms 2024-03-11T20:45:37.850 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:45:38.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243886 10.30 MB/sec execute 441 sec latency 37.014 ms 2024-03-11T20:45:38.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: pgmap v159: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 426 op/s 2024-03-11T20:45:38.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: Upgrade: Updating osd.3 2024-03-11T20:45:38.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:38.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:45:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:45:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:37 smithi083 ceph-mon[66365]: Deploying daemon osd.3 on smithi083 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: pgmap v159: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 426 op/s 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: Upgrade: Updating osd.3 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:45:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:37 smithi204 ceph-mon[53271]: Deploying daemon osd.3 on smithi083 2024-03-11T20:45:38.935 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 27s ago 16m 16.2M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (2m) 27s ago 15m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (2m) 2m ago 15m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 27s ago 15m 45.0M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (9m) 27s ago 9m 39.3M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (9m) 2m ago 9m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (5m) 27s ago 17m 507M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (4m) 2m ago 15m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (4m) 27s ago 17m 49.6M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (3m) 2m ago 15m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (9m) 27s ago 9m 283M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (9m) 2m ago 9m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 27s ago 15m 11.2M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (5m) 2m ago 14m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 27s ago 14m 310M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:45:38.936 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (87s) 27s ago 13m 253M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (38s) 27s ago 13m 12.4M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12m) 27s ago 12m 435M 1936M 17.2.0 e1d6a67b021e 7c57717f362a 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (12m) 2m ago 12m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (12m) 2m ago 12m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (11m) 2m ago 11m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (11m) 2m ago 11m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:45:38.937 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 27s ago 14m 70.0M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:45:39.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244276 10.31 MB/sec execute 442 sec latency 47.585 ms 2024-03-11T20:45:39.236 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:39 smithi083 ceph-mon[66365]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:39.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:39 smithi204 ceph-mon[53271]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:39.568 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 3 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:45:39.569 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 7 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:45:39.570 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:45:40.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:40 smithi083 ceph-mon[66365]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:40.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:40 smithi083 ceph-mon[66365]: pgmap v160: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-03-11T20:45:40.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:40 smithi083 ceph-mon[66365]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:40.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:40 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3572816294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:40.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244567 10.32 MB/sec execute 443 sec latency 52.754 ms 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:45:40.088 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:45:40.089 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:45:40.089 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "9/23 daemons upgraded", 2024-03-11T20:45:40.089 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:45:40.089 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:45:40.089 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:45:40.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:40 smithi204 ceph-mon[53271]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:40.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:40 smithi204 ceph-mon[53271]: pgmap v160: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-03-11T20:45:40.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:40 smithi204 ceph-mon[53271]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:40.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:40 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3572816294' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:45:40.797 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:45:41.036 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:41 smithi083 ceph-mon[66365]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:41.036 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:41 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3864405427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:45:41.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244838 10.32 MB/sec execute 444 sec latency 55.738 ms 2024-03-11T20:45:41.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:41 smithi204 ceph-mon[53271]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:45:41.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:41 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3864405427' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:45:42.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:42 smithi083 ceph-mon[66365]: pgmap v161: 97 pgs: 97 active+clean; 325 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 493 op/s 2024-03-11T20:45:42.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:42.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245066 10.30 MB/sec execute 445 sec latency 48.976 ms 2024-03-11T20:45:42.298 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:42 smithi083 systemd[1]: Stopping Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:45:42.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:42 smithi204 ceph-mon[53271]: pgmap v161: 97 pgs: 97 active+clean; 325 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 493 op/s 2024-03-11T20:45:42.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:42.802 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:42 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[45727]: 2024-03-11T20:45:42.641+0000 7f4c84c6b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:45:42.802 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:42 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[45727]: 2024-03-11T20:45:42.641+0000 7f4c84c6b700 -1 osd.3 104 *** Got signal Terminated *** 2024-03-11T20:45:42.802 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:42 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[45727]: 2024-03-11T20:45:42.641+0000 7f4c84c6b700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:45:43.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245323 10.29 MB/sec execute 446 sec latency 61.617 ms 2024-03-11T20:45:43.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:43 smithi083 ceph-mon[66365]: osd.3 marked itself down and dead 2024-03-11T20:45:43.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:43 smithi204 ceph-mon[53271]: osd.3 marked itself down and dead 2024-03-11T20:45:43.946 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:43 smithi083 podman[79937]: 2024-03-11 20:45:43.689534634 +0000 UTC m=+1.209009768 container died 7c57717f362a1f77ab8b221bdd0c37e46daf6979d9a7df3ef5d684c32c94e326 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., version=8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-03-11T20:45:44.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245323 10.27 MB/sec execute 447 sec latency 1061.653 ms 2024-03-11T20:45:44.290 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:44 smithi083 ceph-mon[66365]: pgmap v162: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-03-11T20:45:44.290 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:44 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:45:44.290 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:44 smithi083 ceph-mon[66365]: osdmap e105: 8 total, 7 up, 8 in 2024-03-11T20:45:44.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:44 smithi204 ceph-mon[53271]: pgmap v162: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-03-11T20:45:44.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:44 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:45:44.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:44 smithi204 ceph-mon[53271]: osdmap e105: 8 total, 7 up, 8 in 2024-03-11T20:45:44.543 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:44 smithi083 podman[79937]: 2024-03-11 20:45:44.363761076 +0000 UTC m=+1.883236216 container cleanup 7c57717f362a1f77ab8b221bdd0c37e46daf6979d9a7df3ef5d684c32c94e326 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, name=centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, version=8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD) 2024-03-11T20:45:44.543 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:44 smithi083 bash[79937]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3 2024-03-11T20:45:44.801 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:44 smithi083 podman[79955]: 2024-03-11 20:45:44.720738787 +0000 UTC m=+1.028733789 container remove 7c57717f362a1f77ab8b221bdd0c37e46daf6979d9a7df3ef5d684c32c94e326 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-03-11T20:45:45.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:45 smithi083 ceph-mon[66365]: osdmap e106: 8 total, 7 up, 8 in 2024-03-11T20:45:45.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245567 10.25 MB/sec execute 448 sec latency 1093.865 ms 2024-03-11T20:45:45.303 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:45 smithi083 podman[80042]: 2024-03-11 20:45:45.153636336 +0000 UTC m=+0.027586298 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:45:45.303 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:45 smithi083 podman[80042]: 2024-03-11 20:45:45.276990942 +0000 UTC m=+0.150940906 container create fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, ceph=True, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:45:45.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:45 smithi204 ceph-mon[53271]: osdmap e106: 8 total, 7 up, 8 in 2024-03-11T20:45:45.830 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:45 smithi083 podman[80042]: 2024-03-11 20:45:45.655875622 +0000 UTC m=+0.529825596 container init fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-11T20:45:45.830 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:45 smithi083 podman[80042]: 2024-03-11 20:45:45.662259577 +0000 UTC m=+0.536209541 container start fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS) 2024-03-11T20:45:45.830 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:45 smithi083 podman[80042]: 2024-03-11 20:45:45.781060031 +0000 UTC m=+0.655010000 container attach fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, io.buildah.version=1.33.5) 2024-03-11T20:45:46.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245817 10.25 MB/sec execute 449 sec latency 45.434 ms 2024-03-11T20:45:46.093 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:46 smithi083 podman[80042]: 2024-03-11 20:45:46.027740085 +0000 UTC m=+0.901690054 container died fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-11T20:45:46.093 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:46 smithi083 ceph-mon[66365]: pgmap v165: 97 pgs: 16 stale+active+clean, 81 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.9 MiB/s wr, 432 op/s 2024-03-11T20:45:46.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:46 smithi204 ceph-mon[53271]: pgmap v165: 97 pgs: 16 stale+active+clean, 81 active+clean; 331 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.9 MiB/s wr, 432 op/s 2024-03-11T20:45:47.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246084 10.24 MB/sec execute 450 sec latency 88.052 ms 2024-03-11T20:45:47.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:47 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 103/1647 objects degraded (6.254%), 15 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:47.123 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 podman[80068]: 2024-03-11 20:45:47.038544787 +0000 UTC m=+0.997907961 container remove fd7db12b503ae9dd2e4b5c5070725d60a1c1ce04a1de612800b9f5ba9e017106 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-11T20:45:47.123 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-11T20:45:47.124 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 systemd[1]: Stopped Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:45:47.124 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service: Consumed 5min 2.714s CPU time. 2024-03-11T20:45:47.392 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 systemd[1]: Starting Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:45:47.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:47 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 103/1647 objects degraded (6.254%), 15 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:47.649 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 podman[80167]: 2024-03-11 20:45:47.521624515 +0000 UTC m=+0.022480819 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:45:47.900 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:47 smithi083 podman[80167]: 2024-03-11 20:45:47.656475241 +0000 UTC m=+0.157331563 container create b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, GIT_CLEAN=True) 2024-03-11T20:45:48.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246381 10.23 MB/sec execute 451 sec latency 65.594 ms 2024-03-11T20:45:48.160 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-mon[66365]: pgmap v166: 97 pgs: 15 active+undersized+degraded, 8 stale+active+clean, 8 active+undersized, 66 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s wr, 346 op/s; 103/1647 objects degraded (6.254%) 2024-03-11T20:45:48.160 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 podman[80167]: 2024-03-11 20:45:48.044552277 +0000 UTC m=+0.545408580 container init b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:45:48.160 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 podman[80167]: 2024-03-11 20:45:48.050059038 +0000 UTC m=+0.550915343 container start b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, ceph=True) 2024-03-11T20:45:48.411 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 podman[80167]: 2024-03-11 20:45:48.218564649 +0000 UTC m=+0.719420961 container attach b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-11T20:45:48.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:48 smithi204 ceph-mon[53271]: pgmap v166: 97 pgs: 15 active+undersized+degraded, 8 stale+active+clean, 8 active+undersized, 66 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 MiB/s wr, 346 op/s; 103/1647 objects degraded (6.254%) 2024-03-11T20:45:48.668 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 bash[80167]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-11T20:45:48.669 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate[80181]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-11T20:45:48.670 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:48 smithi083 podman[80167]: 2024-03-11 20:45:48.60967161 +0000 UTC m=+1.110527927 container died b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux ) 2024-03-11T20:45:49.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246726 10.21 MB/sec execute 452 sec latency 68.253 ms 2024-03-11T20:45:50.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246969 10.20 MB/sec execute 453 sec latency 52.909 ms 2024-03-11T20:45:50.099 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:49 smithi083 podman[80318]: 2024-03-11 20:45:49.879899608 +0000 UTC m=+1.264575205 container remove b316eb1026ba12459e707b944dd6d0fa2d643caeb1a242e6a4a598fd88b40a39 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e) 2024-03-11T20:45:50.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:50 smithi083 ceph-mon[66365]: pgmap v167: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 325 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 361 op/s; 188/1539 objects degraded (12.216%) 2024-03-11T20:45:50.367 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 podman[80380]: 2024-03-11 20:45:50.041782277 +0000 UTC m=+0.022851999 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:45:50.367 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 podman[80380]: 2024-03-11 20:45:50.229889352 +0000 UTC m=+0.210959056 container create 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:45:50.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:50 smithi204 ceph-mon[53271]: pgmap v167: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 325 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 361 op/s; 188/1539 objects degraded (12.216%) 2024-03-11T20:45:50.623 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 podman[80380]: 2024-03-11 20:45:50.602664191 +0000 UTC m=+0.583733892 container init 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-11T20:45:50.623 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 podman[80380]: 2024-03-11 20:45:50.612030708 +0000 UTC m=+0.593100411 container start 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-11T20:45:50.896 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 ceph-osd[80399]: -- 172.21.15.83:0/1255236880 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5584ca9fb680 con 0x5584ca9c5000 2024-03-11T20:45:50.896 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 bash[80380]: 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 2024-03-11T20:45:50.896 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:50 smithi083 systemd[1]: Started Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:45:51.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 247239 10.20 MB/sec execute 454 sec latency 62.840 ms 2024-03-11T20:45:51.411 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:51 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:45:51.183+0000 7f6f049827c0 -1 Falling back to public interface 2024-03-11T20:45:51.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:51.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:51.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:51 smithi083 ceph-mon[66365]: pgmap v168: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 322 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s wr, 350 op/s; 184/1536 objects degraded (11.979%) 2024-03-11T20:45:51.918 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:52.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 247644 10.21 MB/sec execute 455 sec latency 40.884 ms 2024-03-11T20:45:52.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:52.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:52.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:51 smithi204 ceph-mon[53271]: pgmap v168: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 322 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.3 MiB/s wr, 350 op/s; 184/1536 objects degraded (11.979%) 2024-03-11T20:45:52.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:45:52.967 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:52 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 197/1632 objects degraded (12.071%), 26 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:53.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248004 10.21 MB/sec execute 456 sec latency 55.441 ms 2024-03-11T20:45:53.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:52 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 197/1632 objects degraded (12.071%), 26 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:53.982 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:53 smithi083 ceph-mon[66365]: pgmap v169: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 478 op/s; 197/1632 objects degraded (12.071%) 2024-03-11T20:45:54.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248305 10.22 MB/sec execute 457 sec latency 50.461 ms 2024-03-11T20:45:54.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:53 smithi204 ceph-mon[53271]: pgmap v169: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 478 op/s; 197/1632 objects degraded (12.071%) 2024-03-11T20:45:55.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248575 10.22 MB/sec execute 458 sec latency 53.200 ms 2024-03-11T20:45:56.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:55 smithi083 ceph-mon[66365]: pgmap v170: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 434 op/s; 197/1632 objects degraded (12.071%) 2024-03-11T20:45:56.052 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248870 10.21 MB/sec execute 459 sec latency 40.527 ms 2024-03-11T20:45:56.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:55 smithi204 ceph-mon[53271]: pgmap v170: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 434 op/s; 197/1632 objects degraded (12.071%) 2024-03-11T20:45:57.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249079 10.19 MB/sec execute 460 sec latency 85.672 ms 2024-03-11T20:45:57.845 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:57 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:45:57.759+0000 7f6f049827c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-03-11T20:45:57.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:57 smithi083 ceph-mon[66365]: pgmap v171: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.0 MiB/s wr, 413 op/s; 204/1656 objects degraded (12.319%) 2024-03-11T20:45:57.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:57.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:58.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249309 10.19 MB/sec execute 461 sec latency 39.885 ms 2024-03-11T20:45:58.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:57 smithi204 ceph-mon[53271]: pgmap v171: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.0 MiB/s wr, 413 op/s; 204/1656 objects degraded (12.319%) 2024-03-11T20:45:58.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:45:58.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:45:59.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249567 10.18 MB/sec execute 462 sec latency 72.550 ms 2024-03-11T20:45:59.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:58 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 206/1650 objects degraded (12.485%), 26 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:59.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:58 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 206/1650 objects degraded (12.485%), 26 pgs degraded (PG_DEGRADED) 2024-03-11T20:45:59.659 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:45:59.414+0000 7f6f049827c0 -1 osd.3 104 log_to_monitors true 2024-03-11T20:46:00.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249894 10.17 MB/sec execute 463 sec latency 62.574 ms 2024-03-11T20:46:00.162 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:45:59 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:45:59.990+0000 7f6ef5dda700 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:46:00.163 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:59 smithi083 ceph-mon[66365]: pgmap v172: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 206/1650 objects degraded (12.485%) 2024-03-11T20:46:00.163 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:59 smithi083 ceph-mon[66365]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-11T20:46:00.163 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:45:59 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:00.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:59 smithi204 ceph-mon[53271]: pgmap v172: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 206/1650 objects degraded (12.485%) 2024-03-11T20:46:00.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:59 smithi204 ceph-mon[53271]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-11T20:46:00.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:45:59 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:01.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250217 10.15 MB/sec execute 464 sec latency 88.911 ms 2024-03-11T20:46:01.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:00 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:01.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:00 smithi083 ceph-mon[66365]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-11T20:46:01.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:00 smithi083 ceph-mon[66365]: osdmap e107: 8 total, 7 up, 8 in 2024-03-11T20:46:01.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:00 smithi083 ceph-mon[66365]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:46:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:00 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:00 smithi204 ceph-mon[53271]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-11T20:46:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:00 smithi204 ceph-mon[53271]: osdmap e107: 8 total, 7 up, 8 in 2024-03-11T20:46:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:00 smithi204 ceph-mon[53271]: from='osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-03-11T20:46:02.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250217 10.13 MB/sec execute 465 sec latency 1088.948 ms 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: pgmap v174: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 208/1644 objects degraded (12.652%) 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844] boot 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: osdmap e108: 8 total, 8 up, 8 in 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:02.275 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: pgmap v174: 97 pgs: 26 active+undersized+degraded, 15 active+undersized, 56 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 208/1644 objects degraded (12.652%) 2024-03-11T20:46:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:46:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: osd.3 [v2:172.21.15.83:6826/1357805844,v1:172.21.15.83:6827/1357805844] boot 2024-03-11T20:46:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: osdmap e108: 8 total, 8 up, 8 in 2024-03-11T20:46:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-11T20:46:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:03.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250217 10.11 MB/sec execute 466 sec latency 2088.984 ms 2024-03-11T20:46:03.238 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:02 smithi083 ceph-mon[66365]: osdmap e109: 8 total, 8 up, 8 in 2024-03-11T20:46:03.238 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:02 smithi083 ceph-mon[66365]: osdmap e110: 8 total, 8 up, 8 in 2024-03-11T20:46:03.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:03 smithi204 ceph-mon[53271]: osdmap e109: 8 total, 8 up, 8 in 2024-03-11T20:46:03.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:03 smithi204 ceph-mon[53271]: osdmap e110: 8 total, 8 up, 8 in 2024-03-11T20:46:04.023 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:03 smithi083 ceph-mon[66365]: pgmap v177: 97 pgs: 3 remapped+peering, 6 peering, 16 active+undersized+degraded, 8 active+undersized, 64 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 513 op/s; 131/1512 objects degraded (8.664%); 0 B/s, 5 objects/s recovering 2024-03-11T20:46:04.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250450 10.10 MB/sec execute 467 sec latency 2170.436 ms 2024-03-11T20:46:04.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:03 smithi204 ceph-mon[53271]: pgmap v177: 97 pgs: 3 remapped+peering, 6 peering, 16 active+undersized+degraded, 8 active+undersized, 64 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 513 op/s; 131/1512 objects degraded (8.664%); 0 B/s, 5 objects/s recovering 2024-03-11T20:46:05.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250464 10.08 MB/sec execute 468 sec latency 1044.964 ms 2024-03-11T20:46:05.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:05 smithi083 ceph-mon[66365]: osdmap e111: 8 total, 8 up, 8 in 2024-03-11T20:46:05.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:05 smithi204 ceph-mon[53271]: osdmap e111: 8 total, 8 up, 8 in 2024-03-11T20:46:06.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250653 10.07 MB/sec execute 469 sec latency 1029.337 ms 2024-03-11T20:46:06.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:06 smithi083 ceph-mon[66365]: pgmap v180: 97 pgs: 3 remapped+peering, 6 peering, 16 active+undersized+degraded, 8 active+undersized, 64 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 889 KiB/s wr, 100 op/s; 131/1512 objects degraded (8.664%); 0 B/s, 8 objects/s recovering 2024-03-11T20:46:06.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:06 smithi083 ceph-mon[66365]: osdmap e112: 8 total, 8 up, 8 in 2024-03-11T20:46:06.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:06 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 131/1512 objects degraded (8.664%), 16 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:06 smithi204 ceph-mon[53271]: pgmap v180: 97 pgs: 3 remapped+peering, 6 peering, 16 active+undersized+degraded, 8 active+undersized, 64 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 889 KiB/s wr, 100 op/s; 131/1512 objects degraded (8.664%); 0 B/s, 8 objects/s recovering 2024-03-11T20:46:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:06 smithi204 ceph-mon[53271]: osdmap e112: 8 total, 8 up, 8 in 2024-03-11T20:46:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:06 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 131/1512 objects degraded (8.664%), 16 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:07.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250932 10.07 MB/sec execute 470 sec latency 590.179 ms 2024-03-11T20:46:07.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: osdmap e113: 8 total, 8 up, 8 in 2024-03-11T20:46:07.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.144 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: osdmap e113: 8 total, 8 up, 8 in 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:08.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251286 10.08 MB/sec execute 471 sec latency 48.375 ms 2024-03-11T20:46:08.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:08 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:08.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:08 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-11T20:46:08.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:08 smithi083 ceph-mon[66365]: pgmap v183: 97 pgs: 3 remapped+peering, 1 peering, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 943 KiB/s wr, 161 op/s; 12/1443 objects degraded (0.832%); 1.9 MiB/s, 37 objects/s recovering 2024-03-11T20:46:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:08 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:08 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-11T20:46:08.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:08 smithi204 ceph-mon[53271]: pgmap v183: 97 pgs: 3 remapped+peering, 1 peering, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 943 KiB/s wr, 161 op/s; 12/1443 objects degraded (0.832%); 1.9 MiB/s, 37 objects/s recovering 2024-03-11T20:46:09.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251550 10.08 MB/sec execute 472 sec latency 47.178 ms 2024-03-11T20:46:10.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251964 10.09 MB/sec execute 473 sec latency 51.842 ms 2024-03-11T20:46:10.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:10 smithi083 ceph-mon[66365]: pgmap v184: 97 pgs: 1 remapped+peering, 1 peering, 1 active+undersized+degraded, 94 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.7 MiB/s wr, 155 op/s; 12/1536 objects degraded (0.781%); 2.0 MiB/s, 29 objects/s recovering 2024-03-11T20:46:10.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:10 smithi204 ceph-mon[53271]: pgmap v184: 97 pgs: 1 remapped+peering, 1 peering, 1 active+undersized+degraded, 94 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.7 MiB/s wr, 155 op/s; 12/1536 objects degraded (0.781%); 2.0 MiB/s, 29 objects/s recovering 2024-03-11T20:46:11.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252348 10.09 MB/sec execute 474 sec latency 40.488 ms 2024-03-11T20:46:11.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:11 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 12/1536 objects degraded (0.781%), 1 pg degraded (PG_DEGRADED) 2024-03-11T20:46:11.315 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:46:11.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:11 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 12/1536 objects degraded (0.781%), 1 pg degraded (PG_DEGRADED) 2024-03-11T20:46:12.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252650 10.07 MB/sec execute 475 sec latency 44.210 ms 2024-03-11T20:46:12.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:12 smithi083 ceph-mon[66365]: pgmap v185: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 334 op/s; 1.7 MiB/s, 32 objects/s recovering 2024-03-11T20:46:12.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:12 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1536 objects degraded (0.781%), 1 pg degraded) 2024-03-11T20:46:12.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:12 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:46:12.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:12.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 12s ago 16m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (3m) 12s ago 16m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (3m) 3m ago 15m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 12s ago 16m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:46:12.343 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (10m) 12s ago 10m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (10m) 3m ago 10m 16.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (6m) 12s ago 17m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (5m) 3m ago 15m 449M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (4m) 12s ago 17m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (4m) 3m ago 15m 37.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (10m) 12s ago 10m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (10m) 3m ago 10m 89.4M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 12s ago 16m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (6m) 3m ago 15m 15.5M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 12s ago 14m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2m) 12s ago 14m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (71s) 12s ago 13m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:46:12.344 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (22s) 12s ago 13m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:46:12.345 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (13m) 3m ago 13m 559M 2704M 17.2.0 e1d6a67b021e d7b826ee75a5 2024-03-11T20:46:12.345 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (12m) 3m ago 12m 434M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:46:12.345 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (12m) 3m ago 12m 337M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:46:12.345 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (11m) 3m ago 11m 543M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:46:12.345 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 12s ago 15m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:46:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:12 smithi204 ceph-mon[53271]: pgmap v185: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.6 MiB/s wr, 334 op/s; 1.7 MiB/s, 32 objects/s recovering 2024-03-11T20:46:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:12 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1536 objects degraded (0.781%), 1 pg degraded) 2024-03-11T20:46:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:12 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:46:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:12.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:46:12.971 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 4 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:46:12.972 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:46:13.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252883 10.07 MB/sec execute 476 sec latency 40.445 ms 2024-03-11T20:46:13.214 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:13 smithi083 ceph-mon[66365]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:13.214 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:13 smithi083 ceph-mon[66365]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:13.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:13 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1807245485' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:13 smithi204 ceph-mon[53271]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:13 smithi204 ceph-mon[53271]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:13.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:13 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1807245485' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:46:13.505 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "10/23 daemons upgraded", 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:46:13.506 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:46:14.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 253176 10.06 MB/sec execute 477 sec latency 85.779 ms 2024-03-11T20:46:14.117 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:46:14.226 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:14 smithi083 ceph-mon[66365]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:14.226 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:14 smithi083 ceph-mon[66365]: pgmap v186: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 461 op/s; 1.4 MiB/s, 27 objects/s recovering 2024-03-11T20:46:14.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:14 smithi204 ceph-mon[53271]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:14.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:14 smithi204 ceph-mon[53271]: pgmap v186: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 461 op/s; 1.4 MiB/s, 27 objects/s recovering 2024-03-11T20:46:15.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 253554 10.05 MB/sec execute 478 sec latency 46.396 ms 2024-03-11T20:46:15.243 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:15 smithi083 ceph-mon[66365]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:15.243 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:15 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/2731861147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:46:15.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:15 smithi204 ceph-mon[53271]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:15.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:15 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/2731861147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:46:16.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 253861 10.04 MB/sec execute 479 sec latency 66.981 ms 2024-03-11T20:46:16.274 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:16 smithi083 ceph-mon[66365]: pgmap v187: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.0 MiB/s wr, 370 op/s; 1.1 MiB/s, 15 objects/s recovering 2024-03-11T20:46:16.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:16 smithi204 ceph-mon[53271]: pgmap v187: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.0 MiB/s wr, 370 op/s; 1.1 MiB/s, 15 objects/s recovering 2024-03-11T20:46:17.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254186 10.04 MB/sec execute 480 sec latency 88.926 ms 2024-03-11T20:46:18.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:17 smithi083 ceph-mon[66365]: pgmap v188: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 428 op/s; 404 KiB/s, 8 objects/s recovering 2024-03-11T20:46:18.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:18.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254497 10.04 MB/sec execute 481 sec latency 68.225 ms 2024-03-11T20:46:18.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:17 smithi204 ceph-mon[53271]: pgmap v188: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 428 op/s; 404 KiB/s, 8 objects/s recovering 2024-03-11T20:46:18.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:19.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254828 10.04 MB/sec execute 482 sec latency 47.795 ms 2024-03-11T20:46:20.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255128 10.05 MB/sec execute 483 sec latency 61.056 ms 2024-03-11T20:46:20.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:19 smithi083 ceph-mon[66365]: pgmap v189: 97 pgs: 97 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 417 op/s; 350 KiB/s, 5 objects/s recovering 2024-03-11T20:46:20.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:19 smithi204 ceph-mon[53271]: pgmap v189: 97 pgs: 97 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 417 op/s; 350 KiB/s, 5 objects/s recovering 2024-03-11T20:46:21.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255452 10.06 MB/sec execute 484 sec latency 39.226 ms 2024-03-11T20:46:22.053 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255691 10.04 MB/sec execute 485 sec latency 57.121 ms 2024-03-11T20:46:22.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:21 smithi083 ceph-mon[66365]: pgmap v190: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 530 op/s; 8.6 KiB/s, 3 objects/s recovering 2024-03-11T20:46:22.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:22.299 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:21 smithi204 ceph-mon[53271]: pgmap v190: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 530 op/s; 8.6 KiB/s, 3 objects/s recovering 2024-03-11T20:46:22.299 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:23.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255980 10.03 MB/sec execute 486 sec latency 62.623 ms 2024-03-11T20:46:23.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:22 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:23.187 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:22 smithi083 ceph-mon[66365]: Upgrade: osd.4 is safe to restart 2024-03-11T20:46:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:22 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-11T20:46:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:22 smithi204 ceph-mon[53271]: Upgrade: osd.4 is safe to restart 2024-03-11T20:46:24.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 256310 10.03 MB/sec execute 487 sec latency 36.291 ms 2024-03-11T20:46:24.176 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:23 smithi204 ceph-mon[53271]: pgmap v191: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-03-11T20:46:24.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:24.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:46:24.177 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:24.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:23 smithi083 ceph-mon[66365]: pgmap v191: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-03-11T20:46:24.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:24.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-11T20:46:24.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:25.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 256656 10.02 MB/sec execute 488 sec latency 48.544 ms 2024-03-11T20:46:25.200 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:25 smithi204 ceph-mon[53271]: Upgrade: Updating osd.4 2024-03-11T20:46:25.200 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:25 smithi204 ceph-mon[53271]: Deploying daemon osd.4 on smithi204 2024-03-11T20:46:25.234 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:25 smithi083 ceph-mon[66365]: Upgrade: Updating osd.4 2024-03-11T20:46:25.235 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:25 smithi083 ceph-mon[66365]: Deploying daemon osd.4 on smithi204 2024-03-11T20:46:26.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 257092 10.02 MB/sec execute 489 sec latency 64.521 ms 2024-03-11T20:46:26.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:26 smithi204 ceph-mon[53271]: pgmap v192: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 409 op/s 2024-03-11T20:46:26.246 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:26 smithi083 ceph-mon[66365]: pgmap v192: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 409 op/s 2024-03-11T20:46:27.027 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:27.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 257604 10.02 MB/sec execute 490 sec latency 69.677 ms 2024-03-11T20:46:27.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:27.679 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:27 smithi204 systemd[1]: Stopping Ceph osd.4 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:46:28.015 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[31507]: 2024-03-11T20:46:27.696+0000 7f42bbd18700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:46:28.015 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[31507]: 2024-03-11T20:46:27.696+0000 7f42bbd18700 -1 osd.4 113 *** Got signal Terminated *** 2024-03-11T20:46:28.015 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[31507]: 2024-03-11T20:46:27.696+0000 7f42bbd18700 -1 osd.4 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:46:28.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258053 10.03 MB/sec execute 491 sec latency 68.147 ms 2024-03-11T20:46:28.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:28 smithi083 ceph-mon[66365]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 512 op/s 2024-03-11T20:46:28.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:28 smithi083 ceph-mon[66365]: osd.4 marked itself down and dead 2024-03-11T20:46:28.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:28 smithi204 ceph-mon[53271]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 512 op/s 2024-03-11T20:46:28.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:28 smithi204 ceph-mon[53271]: osd.4 marked itself down and dead 2024-03-11T20:46:28.891 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:28 smithi204 podman[57782]: 2024-03-11 20:46:28.633841406 +0000 UTC m=+1.061134743 container died d7b826ee75a5fd6a3a2f5a7b5ec7c7876b0a590a7e8855d96c9b8f4874d028c6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, release=754, ceph=True, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-11T20:46:29.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258058 10.01 MB/sec execute 492 sec latency 1043.918 ms 2024-03-11T20:46:29.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:29 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:46:29.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:29 smithi204 ceph-mon[53271]: osdmap e114: 8 total, 7 up, 8 in 2024-03-11T20:46:29.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:29 smithi204 ceph-mon[53271]: osdmap e115: 8 total, 7 up, 8 in 2024-03-11T20:46:29.179 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:28 smithi204 podman[57782]: 2024-03-11 20:46:28.996107031 +0000 UTC m=+1.423400368 container cleanup d7b826ee75a5fd6a3a2f5a7b5ec7c7876b0a590a7e8855d96c9b8f4874d028c6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, name=centos-stream, release=754, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-11T20:46:29.179 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:28 smithi204 bash[57782]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4 2024-03-11T20:46:29.274 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:29 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:46:29.274 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:29 smithi083 ceph-mon[66365]: osdmap e114: 8 total, 7 up, 8 in 2024-03-11T20:46:29.274 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:29 smithi083 ceph-mon[66365]: osdmap e115: 8 total, 7 up, 8 in 2024-03-11T20:46:29.679 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:29 smithi204 podman[57799]: 2024-03-11 20:46:29.339646513 +0000 UTC m=+0.701039948 container remove d7b826ee75a5fd6a3a2f5a7b5ec7c7876b0a590a7e8855d96c9b8f4874d028c6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-type=git, ceph=True, GIT_CLEAN=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0) 2024-03-11T20:46:30.018 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:29 smithi204 podman[57882]: 2024-03-11 20:46:29.636681228 +0000 UTC m=+0.020481988 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:46:30.018 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:29 smithi204 podman[57882]: 2024-03-11 20:46:29.774234621 +0000 UTC m=+0.158035363 container create 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-11T20:46:30.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258580 10.03 MB/sec execute 493 sec latency 46.983 ms 2024-03-11T20:46:30.137 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:30 smithi083 ceph-mon[66365]: pgmap v195: 97 pgs: 14 stale+active+clean, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 575 op/s 2024-03-11T20:46:30.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:30 smithi204 ceph-mon[53271]: pgmap v195: 97 pgs: 14 stale+active+clean, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 575 op/s 2024-03-11T20:46:30.430 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:30 smithi204 podman[57882]: 2024-03-11 20:46:30.077808885 +0000 UTC m=+0.461609635 container init 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux ) 2024-03-11T20:46:30.430 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:30 smithi204 podman[57882]: 2024-03-11 20:46:30.08146331 +0000 UTC m=+0.465264057 container start 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-11T20:46:30.430 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:30 smithi204 podman[57882]: 2024-03-11 20:46:30.169522387 +0000 UTC m=+0.553323134 container attach 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-2b0101e, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240307) 2024-03-11T20:46:30.897 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:30 smithi204 podman[57882]: 2024-03-11 20:46:30.441063636 +0000 UTC m=+0.824864378 container died 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-11T20:46:31.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258879 10.03 MB/sec execute 494 sec latency 40.610 ms 2024-03-11T20:46:31.148 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:31 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 91/1647 objects degraded (5.525%), 9 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:31.148 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 podman[57908]: 2024-03-11 20:46:31.116493914 +0000 UTC m=+0.669986149 container remove 98bb191ab0baaf33a36caab14a3056c80adfa72e21ceb0f29f04daf164d49dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-11T20:46:31.148 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-11T20:46:31.148 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 systemd[1]: Stopped Ceph osd.4 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:46:31.149 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.4.service: Consumed 4min 53.315s CPU time. 2024-03-11T20:46:31.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:31 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 91/1647 objects degraded (5.525%), 9 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:31.429 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 systemd[1]: Starting Ceph osd.4 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:46:31.899 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 podman[58005]: 2024-03-11 20:46:31.513073693 +0000 UTC m=+0.021497083 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:46:31.899 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 podman[58005]: 2024-03-11 20:46:31.634667576 +0000 UTC m=+0.143091000 container create 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-11T20:46:32.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259205 10.03 MB/sec execute 495 sec latency 39.955 ms 2024-03-11T20:46:32.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-mon[53271]: pgmap v197: 97 pgs: 9 active+undersized+degraded, 7 stale+active+clean, 6 active+undersized, 75 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 458 op/s; 91/1647 objects degraded (5.525%) 2024-03-11T20:46:32.179 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 podman[58005]: 2024-03-11 20:46:31.904537477 +0000 UTC m=+0.412960871 container init 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-11T20:46:32.179 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:31 smithi204 podman[58005]: 2024-03-11 20:46:31.909281716 +0000 UTC m=+0.417705094 container start 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:46:32.179 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 podman[58005]: 2024-03-11 20:46:32.004584849 +0000 UTC m=+0.513008262 container attach 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-11T20:46:32.188 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:32 smithi083 ceph-mon[66365]: pgmap v197: 97 pgs: 9 active+undersized+degraded, 7 stale+active+clean, 6 active+undersized, 75 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 458 op/s; 91/1647 objects degraded (5.525%) 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-11T20:46:32.829 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 bash[58005]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-11T20:46:32.830 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate[58019]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-11T20:46:32.831 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:32 smithi204 podman[58005]: 2024-03-11 20:46:32.471830825 +0000 UTC m=+0.980254194 container died 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-11T20:46:33.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259447 10.02 MB/sec execute 496 sec latency 49.396 ms 2024-03-11T20:46:33.413 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 podman[58141]: 2024-03-11 20:46:33.151570835 +0000 UTC m=+0.673772223 container remove 9835b409ee66091ab27a3fa0788c1fc2de52542608b5400481eaa60172304066 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-11T20:46:33.414 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 podman[58200]: 2024-03-11 20:46:33.288111277 +0000 UTC m=+0.027483170 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:46:33.414 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 podman[58200]: 2024-03-11 20:46:33.41939911 +0000 UTC m=+0.158770976 container create 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-11T20:46:34.054 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259640 10.01 MB/sec execute 497 sec latency 123.276 ms 2024-03-11T20:46:34.087 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 podman[58200]: 2024-03-11 20:46:33.689030206 +0000 UTC m=+0.428402089 container init 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-03-11T20:46:34.087 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 podman[58200]: 2024-03-11 20:46:33.693104698 +0000 UTC m=+0.432476567 container start 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-03-11T20:46:34.087 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 ceph-osd[58217]: -- 172.21.15.204:0/2947594647 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c16dc4f680 con 0x55c16dc19000 2024-03-11T20:46:34.087 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 bash[58200]: 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d 2024-03-11T20:46:34.087 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:33 smithi204 systemd[1]: Started Ceph osd.4 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:46:34.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:34 smithi083 ceph-mon[66365]: pgmap v198: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 644 op/s; 281/1659 objects degraded (16.938%) 2024-03-11T20:46:34.221 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:34 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:34.221 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:34 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:34.221 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:34 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:34.429 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:46:34.240+0000 7fb07db9e7c0 -1 Falling back to public interface 2024-03-11T20:46:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:34 smithi204 ceph-mon[53271]: pgmap v198: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 644 op/s; 281/1659 objects degraded (16.938%) 2024-03-11T20:46:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:34 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:34 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:34 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:35.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259847 10.00 MB/sec execute 498 sec latency 76.794 ms 2024-03-11T20:46:36.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260027 9.99 MB/sec execute 499 sec latency 72.645 ms 2024-03-11T20:46:36.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:36 smithi204 ceph-mon[53271]: pgmap v199: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.8 MiB/s wr, 489 op/s; 281/1659 objects degraded (16.938%) 2024-03-11T20:46:36.295 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:36 smithi083 ceph-mon[66365]: pgmap v199: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.8 MiB/s wr, 489 op/s; 281/1659 objects degraded (16.938%) 2024-03-11T20:46:37.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260362 9.98 MB/sec execute 500 sec latency 65.306 ms 2024-03-11T20:46:37.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:37 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 285/1695 objects degraded (16.814%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:37.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:37 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 285/1695 objects degraded (16.814%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:38.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260708 9.97 MB/sec execute 501 sec latency 47.109 ms 2024-03-11T20:46:38.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:38 smithi083 ceph-mon[66365]: pgmap v200: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 308 op/s; 285/1695 objects degraded (16.814%) 2024-03-11T20:46:38.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:38 smithi204 ceph-mon[53271]: pgmap v200: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 308 op/s; 285/1695 objects degraded (16.814%) 2024-03-11T20:46:39.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261023 9.96 MB/sec execute 502 sec latency 63.389 ms 2024-03-11T20:46:39.354 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:39.354 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:39.355 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:39 smithi083 ceph-mon[66365]: pgmap v201: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 431 op/s; 259/1599 objects degraded (16.198%) 2024-03-11T20:46:39.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:39.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:39.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:39 smithi204 ceph-mon[53271]: pgmap v201: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 431 op/s; 259/1599 objects degraded (16.198%) 2024-03-11T20:46:40.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261330 9.96 MB/sec execute 503 sec latency 47.394 ms 2024-03-11T20:46:40.726 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:40.726 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:40.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:40.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:41.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261660 9.97 MB/sec execute 504 sec latency 63.888 ms 2024-03-11T20:46:41.696 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:41 smithi083 ceph-mon[66365]: pgmap v202: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 324 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 254/1584 objects degraded (16.035%) 2024-03-11T20:46:41.798 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:41 smithi204 ceph-mon[53271]: pgmap v202: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 324 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 254/1584 objects degraded (16.035%) 2024-03-11T20:46:42.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261999 9.97 MB/sec execute 505 sec latency 70.179 ms 2024-03-11T20:46:42.429 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:42 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:46:42.089+0000 7fb07db9e7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-03-11T20:46:42.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:42.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:43.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262229 9.97 MB/sec execute 506 sec latency 62.057 ms 2024-03-11T20:46:43.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:43.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:43.880 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:46:43.683+0000 7fb07db9e7c0 -1 osd.4 113 log_to_monitors true 2024-03-11T20:46:44.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262442 9.96 MB/sec execute 507 sec latency 147.753 ms 2024-03-11T20:46:44.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:43 smithi083 ceph-mon[66365]: pgmap v203: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 322 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 256/1590 objects degraded (16.101%) 2024-03-11T20:46:44.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:43 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:44.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:43 smithi083 ceph-mon[66365]: from='osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:46:44.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:43 smithi083 ceph-mon[66365]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:46:44.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:43 smithi204 ceph-mon[53271]: pgmap v203: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 322 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 256/1590 objects degraded (16.101%) 2024-03-11T20:46:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:43 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:43 smithi204 ceph-mon[53271]: from='osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:46:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:43 smithi204 ceph-mon[53271]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-11T20:46:44.559 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:46:44.679 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:46:44.379+0000 7fb06afee700 -1 osd.4 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:46:45.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262661 9.96 MB/sec execute 508 sec latency 294.092 ms 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: osdmap e116: 8 total, 7 up, 8 in 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='osd.4 ' entity='osd.4' 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.062 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:46:45.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-11T20:46:45.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: osdmap e116: 8 total, 7 up, 8 in 2024-03-11T20:46:45.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:46:45.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='osd.4 ' entity='osd.4' 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 45s ago 17m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (4m) 45s ago 17m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (3m) 7s ago 16m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (7m) 45s ago 16m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (10m) 45s ago 10m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (10m) 7s ago 10m 17.6M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:46:45.517 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (6m) 45s ago 18m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (5m) 7s ago 16m 451M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (5m) 45s ago 18m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (4m) 7s ago 16m 43.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (10m) 45s ago 10m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (10m) 7s ago 10m 89.8M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 45s ago 16m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (6m) 7s ago 16m 18.2M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 45s ago 15m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2m) 45s ago 15m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (104s) 45s ago 14m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (55s) 45s ago 14m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (12s) 7s ago 13m 12.4M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:46:45.518 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (13m) 7s ago 13m 616M 2704M 17.2.0 e1d6a67b021e adef829099b5 2024-03-11T20:46:45.519 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (12m) 7s ago 12m 539M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:46:45.519 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (12m) 7s ago 12m 597M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:46:45.519 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 45s ago 15m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:46:46.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262941 9.95 MB/sec execute 509 sec latency 675.108 ms 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: pgmap v205: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 322 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.9 MiB/s wr, 419 op/s; 256/1590 objects degraded (16.101%) 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995] boot 2024-03-11T20:46:46.077 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: osdmap e117: 8 total, 8 up, 8 in 2024-03-11T20:46:46.078 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:46:46.078 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:45 smithi083 ceph-mon[66365]: osdmap e118: 8 total, 8 up, 8 in 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:46.149 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 5 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 9 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:46:46.150 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: pgmap v205: 97 pgs: 28 active+undersized+degraded, 13 active+undersized, 56 active+clean; 322 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.9 MiB/s wr, 419 op/s; 256/1590 objects degraded (16.101%) 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: osd.4 [v2:172.21.15.204:6800/790951995,v1:172.21.15.204:6801/790951995] boot 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: osdmap e117: 8 total, 8 up, 8 in 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-11T20:46:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:45 smithi204 ceph-mon[53271]: osdmap e118: 8 total, 8 up, 8 in 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:46:46.663 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "11/23 daemons upgraded", 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:46:46.664 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:46:47.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263191 9.94 MB/sec execute 510 sec latency 1116.380 ms 2024-03-11T20:46:47.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:46 smithi083 ceph-mon[66365]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:47.106 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:46 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/429806291' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:47.106 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:46 smithi083 ceph-mon[66365]: osdmap e119: 8 total, 8 up, 8 in 2024-03-11T20:46:47.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:46 smithi204 ceph-mon[53271]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:47.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:46 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/429806291' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:46:47.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:46 smithi204 ceph-mon[53271]: osdmap e119: 8 total, 8 up, 8 in 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 256/1590 objects degraded (16.101%), 28 pgs degraded 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-03-11T20:46:47.304 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-03-11T20:46:47.305 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-03-11T20:46:47.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-03-11T20:46:47.968 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:47 smithi083 ceph-mon[66365]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:47.968 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:47 smithi083 ceph-mon[66365]: pgmap v209: 97 pgs: 1 remapped+peering, 24 active+undersized+degraded, 11 active+undersized, 61 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 97 op/s; 227/1635 objects degraded (13.884%); 21 KiB/s, 68 keys/s, 5 objects/s recovering 2024-03-11T20:46:47.968 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:47 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3551740792' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:46:47.968 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:47 smithi083 ceph-mon[66365]: osdmap e120: 8 total, 8 up, 8 in 2024-03-11T20:46:48.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263553 9.95 MB/sec execute 511 sec latency 518.745 ms 2024-03-11T20:46:48.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:47 smithi204 ceph-mon[53271]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:46:48.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:47 smithi204 ceph-mon[53271]: pgmap v209: 97 pgs: 1 remapped+peering, 24 active+undersized+degraded, 11 active+undersized, 61 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 97 op/s; 227/1635 objects degraded (13.884%); 21 KiB/s, 68 keys/s, 5 objects/s recovering 2024-03-11T20:46:48.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:47 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3551740792' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:46:48.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:47 smithi204 ceph-mon[53271]: osdmap e120: 8 total, 8 up, 8 in 2024-03-11T20:46:49.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263784 9.93 MB/sec execute 512 sec latency 952.046 ms 2024-03-11T20:46:49.655 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:49 smithi083 ceph-mon[66365]: osdmap e121: 8 total, 8 up, 8 in 2024-03-11T20:46:49.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:49 smithi083 ceph-mon[66365]: pgmap v212: 97 pgs: 1 active+undersized+remapped, 1 remapped+peering, 1 peering, 10 active+undersized+degraded, 11 active+undersized, 73 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.0 MiB/s wr, 247 op/s; 112/1683 objects degraded (6.655%); 24 KiB/s, 78 keys/s, 49 objects/s recovering 2024-03-11T20:46:49.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:49 smithi204 ceph-mon[53271]: osdmap e121: 8 total, 8 up, 8 in 2024-03-11T20:46:49.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:49 smithi204 ceph-mon[53271]: pgmap v212: 97 pgs: 1 active+undersized+remapped, 1 remapped+peering, 1 peering, 10 active+undersized+degraded, 11 active+undersized, 73 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.0 MiB/s wr, 247 op/s; 112/1683 objects degraded (6.655%); 24 KiB/s, 78 keys/s, 49 objects/s recovering 2024-03-11T20:46:50.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264166 9.92 MB/sec execute 513 sec latency 279.350 ms 2024-03-11T20:46:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:50 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 112/1683 objects degraded (6.655%), 10 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:50.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:50 smithi083 ceph-mon[66365]: osdmap e122: 8 total, 8 up, 8 in 2024-03-11T20:46:50.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:50 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 112/1683 objects degraded (6.655%), 10 pgs degraded (PG_DEGRADED) 2024-03-11T20:46:50.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:50 smithi204 ceph-mon[53271]: osdmap e122: 8 total, 8 up, 8 in 2024-03-11T20:46:51.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264439 9.91 MB/sec execute 514 sec latency 69.442 ms 2024-03-11T20:46:51.707 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:51 smithi083 ceph-mon[66365]: pgmap v214: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 490 op/s; 990 KiB/s, 65 objects/s recovering 2024-03-11T20:46:51.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:51 smithi204 ceph-mon[53271]: pgmap v214: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 490 op/s; 990 KiB/s, 65 objects/s recovering 2024-03-11T20:46:52.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264806 9.92 MB/sec execute 515 sec latency 49.770 ms 2024-03-11T20:46:52.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:52 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 112/1683 objects degraded (6.655%), 10 pgs degraded) 2024-03-11T20:46:52.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:52 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:46:52.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:52 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 112/1683 objects degraded (6.655%), 10 pgs degraded) 2024-03-11T20:46:52.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:52 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:46:53.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265187 9.92 MB/sec execute 516 sec latency 40.424 ms 2024-03-11T20:46:53.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:53 smithi083 ceph-mon[66365]: pgmap v215: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 593 op/s; 724 KiB/s, 47 objects/s recovering 2024-03-11T20:46:53.830 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:53 smithi204 ceph-mon[53271]: pgmap v215: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 593 op/s; 724 KiB/s, 47 objects/s recovering 2024-03-11T20:46:54.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265585 9.94 MB/sec execute 517 sec latency 46.340 ms 2024-03-11T20:46:55.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265852 9.93 MB/sec execute 518 sec latency 57.746 ms 2024-03-11T20:46:56.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266141 9.92 MB/sec execute 519 sec latency 40.073 ms 2024-03-11T20:46:56.071 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:55 smithi083 ceph-mon[66365]: pgmap v216: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.4 MiB/s wr, 501 op/s; 589 KiB/s, 38 objects/s recovering 2024-03-11T20:46:56.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:55 smithi204 ceph-mon[53271]: pgmap v216: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.4 MiB/s wr, 501 op/s; 589 KiB/s, 38 objects/s recovering 2024-03-11T20:46:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266377 9.91 MB/sec execute 520 sec latency 48.305 ms 2024-03-11T20:46:58.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266630 9.91 MB/sec execute 521 sec latency 94.358 ms 2024-03-11T20:46:58.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:57 smithi083 ceph-mon[66365]: pgmap v217: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.5 MiB/s wr, 460 op/s; 518 KiB/s, 16 objects/s recovering 2024-03-11T20:46:58.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:58.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:57 smithi204 ceph-mon[53271]: pgmap v217: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.5 MiB/s wr, 460 op/s; 518 KiB/s, 16 objects/s recovering 2024-03-11T20:46:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:46:58.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:46:59.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266954 9.91 MB/sec execute 522 sec latency 41.054 ms 2024-03-11T20:47:00.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 267249 9.90 MB/sec execute 523 sec latency 59.588 ms 2024-03-11T20:47:00.134 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:59 smithi083 ceph-mon[66365]: pgmap v218: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 434 KiB/s, 13 objects/s recovering 2024-03-11T20:47:00.135 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:46:59 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:47:00.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:59 smithi204 ceph-mon[53271]: pgmap v218: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 434 KiB/s, 13 objects/s recovering 2024-03-11T20:47:00.152 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:46:59 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:47:00.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:00 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:47:00.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:00 smithi083 ceph-mon[66365]: Upgrade: osd.5 is safe to restart 2024-03-11T20:47:01.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 267590 9.89 MB/sec execute 524 sec latency 67.106 ms 2024-03-11T20:47:01.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:00 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-11T20:47:01.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:00 smithi204 ceph-mon[53271]: Upgrade: osd.5 is safe to restart 2024-03-11T20:47:01.943 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:01 smithi083 ceph-mon[66365]: pgmap v219: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 468 op/s; 207 KiB/s, 6 objects/s recovering 2024-03-11T20:47:01.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:01.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:47:01.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:02.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 267888 9.89 MB/sec execute 525 sec latency 51.308 ms 2024-03-11T20:47:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:01 smithi204 ceph-mon[53271]: pgmap v219: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 468 op/s; 207 KiB/s, 6 objects/s recovering 2024-03-11T20:47:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-11T20:47:02.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:03.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268220 9.88 MB/sec execute 526 sec latency 48.821 ms 2024-03-11T20:47:03.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:02 smithi083 ceph-mon[66365]: Upgrade: Updating osd.5 2024-03-11T20:47:03.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:02 smithi083 ceph-mon[66365]: Deploying daemon osd.5 on smithi204 2024-03-11T20:47:03.239 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:03.247 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:02 smithi204 ceph-mon[53271]: Upgrade: Updating osd.5 2024-03-11T20:47:03.247 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:02 smithi204 ceph-mon[53271]: Deploying daemon osd.5 on smithi204 2024-03-11T20:47:03.247 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:03.999 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:03 smithi083 ceph-mon[66365]: pgmap v220: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 535 op/s 2024-03-11T20:47:04.055 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268589 9.89 MB/sec execute 527 sec latency 48.250 ms 2024-03-11T20:47:04.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:03 smithi204 ceph-mon[53271]: pgmap v220: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 535 op/s 2024-03-11T20:47:05.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268979 9.90 MB/sec execute 528 sec latency 35.368 ms 2024-03-11T20:47:05.430 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:05 smithi204 systemd[1]: Stopping Ceph osd.5 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:47:05.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:05 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[34472]: 2024-03-11T20:47:05.442+0000 7f9b159f7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:47:05.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:05 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[34472]: 2024-03-11T20:47:05.442+0000 7f9b159f7700 -1 osd.5 122 *** Got signal Terminated *** 2024-03-11T20:47:05.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:05 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[34472]: 2024-03-11T20:47:05.442+0000 7f9b159f7700 -1 osd.5 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:47:06.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269293 9.90 MB/sec execute 529 sec latency 64.180 ms 2024-03-11T20:47:06.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:06 smithi083 ceph-mon[66365]: pgmap v221: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 427 op/s 2024-03-11T20:47:06.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:06 smithi083 ceph-mon[66365]: osd.5 marked itself down and dead 2024-03-11T20:47:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:06 smithi204 ceph-mon[53271]: pgmap v221: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 427 op/s 2024-03-11T20:47:06.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:06 smithi204 ceph-mon[53271]: osd.5 marked itself down and dead 2024-03-11T20:47:06.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:06 smithi204 podman[60017]: 2024-03-11 20:47:06.684331851 +0000 UTC m=+1.388569718 container died adef829099b5f46b484a383346cb0055a0810100be685a481fb3499e656ec4b3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, version=8, architecture=x86_64, io.openshift.expose-services=, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container) 2024-03-11T20:47:07.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269296 9.88 MB/sec execute 530 sec latency 1057.319 ms 2024-03-11T20:47:07.253 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:07 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:47:07.254 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:07 smithi083 ceph-mon[66365]: osdmap e123: 8 total, 7 up, 8 in 2024-03-11T20:47:07.346 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:07 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:47:07.347 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:07 smithi204 ceph-mon[53271]: osdmap e123: 8 total, 7 up, 8 in 2024-03-11T20:47:07.347 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60017]: 2024-03-11 20:47:07.049626142 +0000 UTC m=+1.753864017 container cleanup adef829099b5f46b484a383346cb0055a0810100be685a481fb3499e656ec4b3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, release=754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-03-11T20:47:07.347 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 bash[60017]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5 2024-03-11T20:47:07.682 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60033]: 2024-03-11 20:47:07.352230608 +0000 UTC m=+0.665262948 container remove adef829099b5f46b484a383346cb0055a0810100be685a481fb3499e656ec4b3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-03-11T20:47:07.976 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60125]: 2024-03-11 20:47:07.62875672 +0000 UTC m=+0.021685902 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:07.976 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60125]: 2024-03-11 20:47:07.753415889 +0000 UTC m=+0.146345055 container create 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:47:08.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269558 9.87 MB/sec execute 531 sec latency 1083.368 ms 2024-03-11T20:47:08.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:08 smithi083 ceph-mon[66365]: pgmap v223: 97 pgs: 4 peering, 11 stale+active+clean, 82 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 515 op/s 2024-03-11T20:47:08.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:08 smithi083 ceph-mon[66365]: osdmap e124: 8 total, 7 up, 8 in 2024-03-11T20:47:08.326 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60125]: 2024-03-11 20:47:07.981980559 +0000 UTC m=+0.374909749 container init 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-11T20:47:08.326 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:07 smithi204 podman[60125]: 2024-03-11 20:47:07.986486715 +0000 UTC m=+0.379415886 container start 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-11T20:47:08.326 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:08 smithi204 podman[60125]: 2024-03-11 20:47:08.081843958 +0000 UTC m=+0.474773130 container attach 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, RELEASE=main-2b0101e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-11T20:47:08.326 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:08 smithi204 ceph-mon[53271]: pgmap v223: 97 pgs: 4 peering, 11 stale+active+clean, 82 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 515 op/s 2024-03-11T20:47:08.327 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:08 smithi204 ceph-mon[53271]: osdmap e124: 8 total, 7 up, 8 in 2024-03-11T20:47:08.679 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:08 smithi204 podman[60125]: 2024-03-11 20:47:08.331670563 +0000 UTC m=+0.724599743 container died 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-03-11T20:47:09.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269850 9.87 MB/sec execute 532 sec latency 37.970 ms 2024-03-11T20:47:09.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:09 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 66/1632 objects degraded (4.044%), 8 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:09.399 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:09 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 66/1632 objects degraded (4.044%), 8 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:09.399 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 podman[60151]: 2024-03-11 20:47:09.129030132 +0000 UTC m=+0.789064705 container remove 0540c56d46133ec98b6995293da78c47991e4dc52fb12d457362f7198abd2b9b (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-11T20:47:09.399 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.5.service: Deactivated successfully. 2024-03-11T20:47:09.399 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 systemd[1]: Stopped Ceph osd.5 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:47:09.399 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.5.service: Consumed 5min 39.636s CPU time. 2024-03-11T20:47:09.679 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 systemd[1]: Starting Ceph osd.5 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:47:09.679 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 podman[60253]: 2024-03-11 20:47:09.546032428 +0000 UTC m=+0.021173456 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:10.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270125 9.86 MB/sec execute 533 sec latency 39.169 ms 2024-03-11T20:47:10.079 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:09 smithi204 podman[60253]: 2024-03-11 20:47:09.822001319 +0000 UTC m=+0.297142344 container create 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-11T20:47:10.079 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 podman[60253]: 2024-03-11 20:47:10.042049301 +0000 UTC m=+0.517190327 container init 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=main-2b0101e, ceph=True, org.label-schema.license=GPLv2) 2024-03-11T20:47:10.079 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 podman[60253]: 2024-03-11 20:47:10.045412705 +0000 UTC m=+0.520553739 container start 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-11T20:47:10.158 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:10 smithi083 ceph-mon[66365]: pgmap v225: 97 pgs: 8 active+undersized+degraded, 4 peering, 7 stale+active+clean, 3 active+undersized, 75 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 485 op/s; 66/1632 objects degraded (4.044%) 2024-03-11T20:47:10.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-mon[53271]: pgmap v225: 97 pgs: 8 active+undersized+degraded, 4 peering, 7 stale+active+clean, 3 active+undersized, 75 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 485 op/s; 66/1632 objects degraded (4.044%) 2024-03-11T20:47:10.430 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 podman[60253]: 2024-03-11 20:47:10.14225134 +0000 UTC m=+0.617392380 container attach 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:47:10.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 bash[60253]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate[60268]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-11T20:47:10.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:10 smithi204 podman[60253]: 2024-03-11 20:47:10.644211178 +0000 UTC m=+1.119352214 container died 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-11T20:47:11.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270433 9.86 MB/sec execute 534 sec latency 43.603 ms 2024-03-11T20:47:11.551 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 podman[60394]: 2024-03-11 20:47:11.30574125 +0000 UTC m=+0.655218048 container remove 6319da21685d30189ad12c4a340e586e67e8d56da743aa3d2cd64c546a35573c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, io.buildah.version=1.33.5) 2024-03-11T20:47:11.812 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 podman[60451]: 2024-03-11 20:47:11.460543353 +0000 UTC m=+0.020276197 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:11.813 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 podman[60451]: 2024-03-11 20:47:11.581876169 +0000 UTC m=+0.141609003 container create e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:47:11.813 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 podman[60451]: 2024-03-11 20:47:11.818361712 +0000 UTC m=+0.378094547 container init e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-2b0101e, ceph=True, GIT_BRANCH=HEAD) 2024-03-11T20:47:12.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270782 9.86 MB/sec execute 535 sec latency 41.295 ms 2024-03-11T20:47:12.081 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 podman[60451]: 2024-03-11 20:47:11.822068272 +0000 UTC m=+0.381801104 container start e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_CLEAN=True) 2024-03-11T20:47:12.081 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 ceph-osd[60470]: -- 172.21.15.204:0/2978538233 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564b02f8d680 con 0x564b02f57000 2024-03-11T20:47:12.081 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 bash[60451]: e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 2024-03-11T20:47:12.081 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:11 smithi204 systemd[1]: Started Ceph osd.5 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:47:12.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: pgmap v226: 97 pgs: 19 active+undersized+degraded, 4 peering, 8 active+undersized, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.5 MiB/s wr, 385 op/s; 159/1692 objects degraded (9.397%) 2024-03-11T20:47:12.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:12.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.191 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: pgmap v226: 97 pgs: 19 active+undersized+degraded, 4 peering, 8 active+undersized, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.5 MiB/s wr, 385 op/s; 159/1692 objects degraded (9.397%) 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:12.362 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:12.679 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:12 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:47:12.366+0000 7fd35e6607c0 -1 Falling back to public interface 2024-03-11T20:47:13.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 271139 9.85 MB/sec execute 536 sec latency 51.942 ms 2024-03-11T20:47:14.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 271496 9.84 MB/sec execute 537 sec latency 37.113 ms 2024-03-11T20:47:14.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:14 smithi083 ceph-mon[66365]: pgmap v227: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 488 op/s; 171/1668 objects degraded (10.252%) 2024-03-11T20:47:14.298 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:14 smithi204 ceph-mon[53271]: pgmap v227: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 488 op/s; 171/1668 objects degraded (10.252%) 2024-03-11T20:47:15.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 271955 9.85 MB/sec execute 538 sec latency 55.707 ms 2024-03-11T20:47:15.263 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:15 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 171/1680 objects degraded (10.179%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:15.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:15 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 171/1680 objects degraded (10.179%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:16.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 272424 9.87 MB/sec execute 539 sec latency 40.964 ms 2024-03-11T20:47:16.270 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:16 smithi083 ceph-mon[66365]: pgmap v228: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.6 MiB/s wr, 349 op/s; 171/1680 objects degraded (10.179%) 2024-03-11T20:47:16.408 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:16 smithi204 ceph-mon[53271]: pgmap v228: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.6 MiB/s wr, 349 op/s; 171/1680 objects degraded (10.179%) 2024-03-11T20:47:17.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 272878 9.88 MB/sec execute 540 sec latency 37.525 ms 2024-03-11T20:47:17.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:17.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:17.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:17 smithi083 ceph-mon[66365]: pgmap v229: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 171/1674 objects degraded (10.215%) 2024-03-11T20:47:17.810 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:47:18.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 273159 9.87 MB/sec execute 541 sec latency 57.888 ms 2024-03-11T20:47:18.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:18.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:18.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:17 smithi204 ceph-mon[53271]: pgmap v229: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 171/1674 objects degraded (10.215%) 2024-03-11T20:47:18.715 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:47:18.715 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (8m) 78s ago 17m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:47:18.715 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (4m) 78s ago 17m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (4m) 2s ago 16m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (8m) 78s ago 17m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (11m) 78s ago 11m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (11m) 2s ago 11m 17.7M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (7m) 78s ago 18m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (6m) 2s ago 16m 452M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 78s ago 19m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (5m) 2s ago 16m 44.7M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (11m) 78s ago 11m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (11m) 2s ago 11m 89.8M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 78s ago 17m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (7m) 2s ago 16m 18.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 78s ago 16m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 78s ago 15m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:47:18.716 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 78s ago 15m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (88s) 78s ago 14m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (45s) 2s ago 14m 265M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (7s) 2s ago 13m 12.7M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (13m) 2s ago 13m 579M 2704M 17.2.0 e1d6a67b021e c96524c962b7 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (13m) 2s ago 13m 648M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:47:18.717 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 78s ago 16m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:47:19.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 273413 9.86 MB/sec execute 542 sec latency 58.612 ms 2024-03-11T20:47:19.270 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:47:19.270 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:47:19.270 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 5 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-11T20:47:19.271 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 9 2024-03-11T20:47:19.272 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:47:19.272 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:47:19.322 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:19 smithi083 ceph-mon[66365]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:19.323 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:19.323 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:19 smithi204 ceph-mon[53271]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:19.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:19.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:47:19.863 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:47:19.864 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "12/23 daemons upgraded", 2024-03-11T20:47:19.864 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:47:19.864 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:47:19.864 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:47:20.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 273696 9.86 MB/sec execute 543 sec latency 36.031 ms 2024-03-11T20:47:20.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:20 smithi204 ceph-mon[53271]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:20 smithi204 ceph-mon[53271]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:20 smithi204 ceph-mon[53271]: pgmap v230: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 379 op/s; 172/1689 objects degraded (10.184%) 2024-03-11T20:47:20.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:20 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3911126950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:20.359 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:20 smithi083 ceph-mon[66365]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:20.359 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:20 smithi083 ceph-mon[66365]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:20.359 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:20 smithi083 ceph-mon[66365]: pgmap v230: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 379 op/s; 172/1689 objects degraded (10.184%) 2024-03-11T20:47:20.359 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:20 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3911126950' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:20.492 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 172/1689 objects degraded (10.184%), 22 pgs degraded 2024-03-11T20:47:20.492 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: osd.5 (root=default,host=smithi204) is down 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 172/1689 objects degraded (10.184%), 22 pgs degraded 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-03-11T20:47:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-03-11T20:47:20.494 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-03-11T20:47:21.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 274033 9.86 MB/sec execute 544 sec latency 50.299 ms 2024-03-11T20:47:21.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:21 smithi083 ceph-mon[66365]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:21.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:21 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 172/1689 objects degraded (10.184%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:21.115 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:21 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/550052161' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:47:21.347 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:21 smithi204 ceph-mon[53271]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:21.347 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:21 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 172/1689 objects degraded (10.184%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:21.347 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:21 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/550052161' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:47:21.624 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:21 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:47:21.551+0000 7fd35e6607c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-03-11T20:47:22.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 274385 9.86 MB/sec execute 545 sec latency 44.363 ms 2024-03-11T20:47:22.134 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:22 smithi083 ceph-mon[66365]: pgmap v231: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 574 op/s; 168/1737 objects degraded (9.672%) 2024-03-11T20:47:22.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:22 smithi204 ceph-mon[53271]: pgmap v231: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 574 op/s; 168/1737 objects degraded (9.672%) 2024-03-11T20:47:23.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 274803 9.85 MB/sec execute 546 sec latency 68.482 ms 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:23.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:47:23.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:23 smithi083 ceph-mon[66365]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:47:23.330 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:47:22.947+0000 7fd35e6607c0 -1 osd.5 122 log_to_monitors true 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:23.331 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:23.332 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:47:23.332 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-mon[53271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-11T20:47:23.929 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:47:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:47:23.639+0000 7fd34bab0700 -1 osd.5 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:47:24.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 275030 9.84 MB/sec execute 547 sec latency 143.370 ms 2024-03-11T20:47:24.195 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: pgmap v232: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 470 op/s; 172/1755 objects degraded (9.801%) 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: from='osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: osdmap e125: 8 total, 7 up, 8 in 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:47:24.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:24 smithi083 ceph-mon[66365]: from='osd.5 ' entity='osd.5' 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: pgmap v232: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 470 op/s; 172/1755 objects degraded (9.801%) 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: from='osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: osdmap e125: 8 total, 7 up, 8 in 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:47:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:24 smithi204 ceph-mon[53271]: from='osd.5 ' entity='osd.5' 2024-03-11T20:47:25.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 275222 9.83 MB/sec execute 548 sec latency 423.860 ms 2024-03-11T20:47:25.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:25 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:47:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:25 smithi204 ceph-mon[53271]: osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408] boot 2024-03-11T20:47:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:25 smithi204 ceph-mon[53271]: osdmap e126: 8 total, 8 up, 8 in 2024-03-11T20:47:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:47:25.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:25 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:47:25.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:25 smithi083 ceph-mon[66365]: osd.5 [v2:172.21.15.204:6808/1942466408,v1:172.21.15.204:6809/1942466408] boot 2024-03-11T20:47:25.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:25 smithi083 ceph-mon[66365]: osdmap e126: 8 total, 8 up, 8 in 2024-03-11T20:47:25.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-11T20:47:26.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 275418 9.83 MB/sec execute 549 sec latency 1321.034 ms 2024-03-11T20:47:26.320 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:26 smithi083 ceph-mon[66365]: pgmap v235: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 174/1722 objects degraded (10.105%) 2024-03-11T20:47:26.321 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:26 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 172/1755 objects degraded (9.801%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:26.321 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:26 smithi083 ceph-mon[66365]: osdmap e127: 8 total, 8 up, 8 in 2024-03-11T20:47:26.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:26 smithi204 ceph-mon[53271]: pgmap v235: 97 pgs: 22 active+undersized+degraded, 9 active+undersized, 66 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 174/1722 objects degraded (10.105%) 2024-03-11T20:47:26.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:26 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 172/1755 objects degraded (9.801%), 22 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:26.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:26 smithi204 ceph-mon[53271]: osdmap e127: 8 total, 8 up, 8 in 2024-03-11T20:47:27.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 275561 9.82 MB/sec execute 550 sec latency 2258.139 ms 2024-03-11T20:47:27.486 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:27 smithi083 ceph-mon[66365]: osdmap e128: 8 total, 8 up, 8 in 2024-03-11T20:47:27.487 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:27 smithi083 ceph-mon[66365]: pgmap v238: 97 pgs: 1 unknown, 6 peering, 16 active+undersized+degraded, 1 active+recovery_wait+degraded, 5 active+undersized, 68 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 516 op/s; 136/1623 objects degraded (8.380%); 22 B/s, 0 objects/s recovering 2024-03-11T20:47:27.487 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:27.487 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:27 smithi204 ceph-mon[53271]: osdmap e128: 8 total, 8 up, 8 in 2024-03-11T20:47:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:27 smithi204 ceph-mon[53271]: pgmap v238: 97 pgs: 1 unknown, 6 peering, 16 active+undersized+degraded, 1 active+recovery_wait+degraded, 5 active+undersized, 68 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 516 op/s; 136/1623 objects degraded (8.380%); 22 B/s, 0 objects/s recovering 2024-03-11T20:47:27.929 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:27.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:28.056 INFO:teuthology.orchestra.run.smithi083.stdout: 5 275971 9.82 MB/sec execute 551 sec latency 172.973 ms 2024-03-11T20:47:29.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:28 smithi083 ceph-mon[66365]: osdmap e129: 8 total, 8 up, 8 in 2024-03-11T20:47:29.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 276297 9.82 MB/sec execute 552 sec latency 1032.844 ms 2024-03-11T20:47:29.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:28 smithi204 ceph-mon[53271]: osdmap e129: 8 total, 8 up, 8 in 2024-03-11T20:47:30.014 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:29 smithi083 ceph-mon[66365]: pgmap v240: 97 pgs: 85 active+clean, 2 active+undersized, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 7 peering, 1 unknown; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 528 op/s; 20/1593 objects degraded (1.255%); 1.1 MiB/s, 37 objects/s recovering 2024-03-11T20:47:30.015 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:29 smithi083 ceph-mon[66365]: osdmap e130: 8 total, 8 up, 8 in 2024-03-11T20:47:30.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 276561 9.82 MB/sec execute 553 sec latency 36.847 ms 2024-03-11T20:47:30.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:29 smithi204 ceph-mon[53271]: pgmap v240: 97 pgs: 85 active+clean, 2 active+undersized, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 7 peering, 1 unknown; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 528 op/s; 20/1593 objects degraded (1.255%); 1.1 MiB/s, 37 objects/s recovering 2024-03-11T20:47:30.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:29 smithi204 ceph-mon[53271]: osdmap e130: 8 total, 8 up, 8 in 2024-03-11T20:47:31.031 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:30 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 20/1593 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2024-03-11T20:47:31.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 276927 9.82 MB/sec execute 554 sec latency 32.038 ms 2024-03-11T20:47:31.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:30 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 20/1593 objects degraded (1.255%), 1 pg degraded (PG_DEGRADED) 2024-03-11T20:47:32.056 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:31 smithi083 ceph-mon[66365]: pgmap v242: 97 pgs: 94 active+clean, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 peering; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 358 op/s; 20/1647 objects degraded (1.214%); 1.3 MiB/s, 42 objects/s recovering 2024-03-11T20:47:32.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 277309 9.82 MB/sec execute 555 sec latency 54.534 ms 2024-03-11T20:47:32.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:31 smithi204 ceph-mon[53271]: pgmap v242: 97 pgs: 94 active+clean, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 peering; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 358 op/s; 20/1647 objects degraded (1.214%); 1.3 MiB/s, 42 objects/s recovering 2024-03-11T20:47:33.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 277619 9.82 MB/sec execute 556 sec latency 77.468 ms 2024-03-11T20:47:33.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:32 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1647 objects degraded (1.214%), 1 pg degraded) 2024-03-11T20:47:33.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:32 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:47:33.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:32 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1647 objects degraded (1.214%), 1 pg degraded) 2024-03-11T20:47:33.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:32 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:47:34.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 277959 9.82 MB/sec execute 557 sec latency 39.174 ms 2024-03-11T20:47:34.086 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:33 smithi083 ceph-mon[66365]: pgmap v243: 97 pgs: 96 active+clean, 1 peering; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 404 op/s; 1.1 MiB/s, 38 objects/s recovering 2024-03-11T20:47:34.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:33 smithi204 ceph-mon[53271]: pgmap v243: 97 pgs: 96 active+clean, 1 peering; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 404 op/s; 1.1 MiB/s, 38 objects/s recovering 2024-03-11T20:47:35.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 278377 9.81 MB/sec execute 558 sec latency 44.389 ms 2024-03-11T20:47:36.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 278731 9.81 MB/sec execute 559 sec latency 42.056 ms 2024-03-11T20:47:36.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:35 smithi083 ceph-mon[66365]: pgmap v244: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 355 op/s; 912 KiB/s, 31 objects/s recovering 2024-03-11T20:47:36.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:35 smithi204 ceph-mon[53271]: pgmap v244: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 355 op/s; 912 KiB/s, 31 objects/s recovering 2024-03-11T20:47:37.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 279096 9.82 MB/sec execute 560 sec latency 52.462 ms 2024-03-11T20:47:38.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 279497 9.82 MB/sec execute 561 sec latency 65.861 ms 2024-03-11T20:47:38.179 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:37 smithi083 ceph-mon[66365]: pgmap v245: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 343 KiB/s, 23 objects/s recovering 2024-03-11T20:47:38.180 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:38.270 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:37 smithi204 ceph-mon[53271]: pgmap v245: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.3 MiB/s wr, 415 op/s; 343 KiB/s, 23 objects/s recovering 2024-03-11T20:47:38.270 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:39.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 279936 9.82 MB/sec execute 562 sec latency 63.898 ms 2024-03-11T20:47:39.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:38 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:39.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:38 smithi083 ceph-mon[66365]: Upgrade: osd.6 is safe to restart 2024-03-11T20:47:39.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:38 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-11T20:47:39.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:38 smithi204 ceph-mon[53271]: Upgrade: osd.6 is safe to restart 2024-03-11T20:47:40.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 280249 9.82 MB/sec execute 563 sec latency 37.402 ms 2024-03-11T20:47:40.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:40 smithi083 ceph-mon[66365]: pgmap v246: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 471 op/s; 274 KiB/s, 9 objects/s recovering 2024-03-11T20:47:40.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:40 smithi083 ceph-mon[66365]: Upgrade: Updating osd.6 2024-03-11T20:47:40.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:40.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:47:40.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:40.556 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:40 smithi204 ceph-mon[53271]: pgmap v246: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 471 op/s; 274 KiB/s, 9 objects/s recovering 2024-03-11T20:47:40.556 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:40 smithi204 ceph-mon[53271]: Upgrade: Updating osd.6 2024-03-11T20:47:40.556 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:40.556 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-11T20:47:40.557 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:47:41.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 280671 9.82 MB/sec execute 564 sec latency 52.234 ms 2024-03-11T20:47:41.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:41 smithi083 ceph-mon[66365]: Deploying daemon osd.6 on smithi204 2024-03-11T20:47:41.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:41 smithi204 ceph-mon[53271]: Deploying daemon osd.6 on smithi204 2024-03-11T20:47:42.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 280981 9.82 MB/sec execute 565 sec latency 49.810 ms 2024-03-11T20:47:42.305 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:42 smithi204 ceph-mon[53271]: pgmap v247: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 548 op/s; 229 KiB/s, 7 objects/s recovering 2024-03-11T20:47:42.305 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:42.305 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:42.305 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:42.424 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:42 smithi083 ceph-mon[66365]: pgmap v247: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 548 op/s; 229 KiB/s, 7 objects/s recovering 2024-03-11T20:47:42.425 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:42.425 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:42.425 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:43.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 281287 9.82 MB/sec execute 566 sec latency 48.792 ms 2024-03-11T20:47:43.405 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:43 smithi204 ceph-mon[53271]: pgmap v248: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 463 op/s; 2.2 KiB/s, 1 objects/s recovering 2024-03-11T20:47:43.451 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:43 smithi083 ceph-mon[66365]: pgmap v248: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 463 op/s; 2.2 KiB/s, 1 objects/s recovering 2024-03-11T20:47:43.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:43 smithi204 systemd[1]: Stopping Ceph osd.6 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:47:44.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 281567 9.81 MB/sec execute 567 sec latency 54.678 ms 2024-03-11T20:47:44.179 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[37436]: 2024-03-11T20:47:43.715+0000 7f30a2cf9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:47:44.179 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[37436]: 2024-03-11T20:47:43.715+0000 7f30a2cf9700 -1 osd.6 130 *** Got signal Terminated *** 2024-03-11T20:47:44.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[37436]: 2024-03-11T20:47:43.715+0000 7f30a2cf9700 -1 osd.6 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:47:44.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:44 smithi083 ceph-mon[66365]: osd.6 marked itself down and dead 2024-03-11T20:47:44.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:44 smithi204 ceph-mon[53271]: osd.6 marked itself down and dead 2024-03-11T20:47:45.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 281739 9.80 MB/sec execute 568 sec latency 727.696 ms 2024-03-11T20:47:45.145 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:44 smithi204 podman[62280]: 2024-03-11 20:47:44.864162248 +0000 UTC m=+1.279235486 container died c96524c962b76845e2360e9d5b3668ea7ef325f86ce41d3494770eb5d1bba032 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754) 2024-03-11T20:47:45.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:45 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:47:45.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:45 smithi204 ceph-mon[53271]: osdmap e131: 8 total, 7 up, 8 in 2024-03-11T20:47:45.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:45 smithi204 ceph-mon[53271]: pgmap v250: 97 pgs: 10 stale+active+clean, 87 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 538 op/s 2024-03-11T20:47:45.430 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:45 smithi204 podman[62280]: 2024-03-11 20:47:45.225151574 +0000 UTC m=+1.640224829 container cleanup c96524c962b76845e2360e9d5b3668ea7ef325f86ce41d3494770eb5d1bba032 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, vcs-type=git, io.openshift.tags=base centos centos-stream, version=8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-03-11T20:47:45.430 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:45 smithi204 bash[62280]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6 2024-03-11T20:47:45.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:45 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:47:45.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:45 smithi083 ceph-mon[66365]: osdmap e131: 8 total, 7 up, 8 in 2024-03-11T20:47:45.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:45 smithi083 ceph-mon[66365]: pgmap v250: 97 pgs: 10 stale+active+clean, 87 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 538 op/s 2024-03-11T20:47:45.861 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:45 smithi204 podman[62297]: 2024-03-11 20:47:45.518139226 +0000 UTC m=+0.653070770 container remove c96524c962b76845e2360e9d5b3668ea7ef325f86ce41d3494770eb5d1bba032 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-type=git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., name=centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-03-11T20:47:46.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 282048 9.79 MB/sec execute 569 sec latency 911.836 ms 2024-03-11T20:47:46.149 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:45 smithi204 podman[62383]: 2024-03-11 20:47:45.769913811 +0000 UTC m=+0.021992175 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:46.150 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:46 smithi204 podman[62383]: 2024-03-11 20:47:46.027792622 +0000 UTC m=+0.279870973 container create 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:47:46.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:46 smithi083 ceph-mon[66365]: osdmap e132: 8 total, 7 up, 8 in 2024-03-11T20:47:46.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:46 smithi204 ceph-mon[53271]: osdmap e132: 8 total, 7 up, 8 in 2024-03-11T20:47:46.429 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:46 smithi204 podman[62383]: 2024-03-11 20:47:46.289714547 +0000 UTC m=+0.541792919 container init 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-11T20:47:46.430 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:46 smithi204 podman[62383]: 2024-03-11 20:47:46.293212259 +0000 UTC m=+0.545290616 container start 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307) 2024-03-11T20:47:46.430 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:46 smithi204 podman[62383]: 2024-03-11 20:47:46.38953577 +0000 UTC m=+0.641614130 container attach 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:47:46.929 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:46 smithi204 podman[62383]: 2024-03-11 20:47:46.644769365 +0000 UTC m=+0.896847712 container died 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:47:47.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 282302 9.79 MB/sec execute 570 sec latency 50.047 ms 2024-03-11T20:47:47.255 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:47 smithi083 ceph-mon[66365]: pgmap v252: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 8 peering, 7 stale+active+clean, 78 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.6 MiB/s wr, 584 op/s; 20/1635 objects degraded (1.223%) 2024-03-11T20:47:47.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:47 smithi204 ceph-mon[53271]: pgmap v252: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 8 peering, 7 stale+active+clean, 78 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.6 MiB/s wr, 584 op/s; 20/1635 objects degraded (1.223%) 2024-03-11T20:47:47.929 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:47 smithi204 podman[62408]: 2024-03-11 20:47:47.578721129 +0000 UTC m=+0.927952713 container remove 9edb7f9b7b542ddc1e654c4feb17a87f23f05ebfd9f9ae02ebc06182a0ce5f95 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.build-date=20240307) 2024-03-11T20:47:47.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:47 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.6.service: Deactivated successfully. 2024-03-11T20:47:47.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:47 smithi204 systemd[1]: Stopped Ceph osd.6 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:47:47.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:47 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.6.service: Consumed 5min 24.460s CPU time. 2024-03-11T20:47:47.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:47 smithi204 systemd[1]: Starting Ceph osd.6 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:47:48.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 282600 9.79 MB/sec execute 571 sec latency 44.312 ms 2024-03-11T20:47:48.288 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:48 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 20/1635 objects degraded (1.223%), 3 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:48.328 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 20/1635 objects degraded (1.223%), 3 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:48.329 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:47.954918265 +0000 UTC m=+0.021708979 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:48.329 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:48.080195293 +0000 UTC m=+0.146985981 container create 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-03-11T20:47:48.679 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:48.333858013 +0000 UTC m=+0.400648727 container init 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:47:48.679 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:48.341559111 +0000 UTC m=+0.408349804 container start 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, GIT_CLEAN=True) 2024-03-11T20:47:48.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:48.425466028 +0000 UTC m=+0.492256732 container attach 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:47:49.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 282957 9.79 MB/sec execute 572 sec latency 59.139 ms 2024-03-11T20:47:49.179 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 bash[62515]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-11T20:47:49.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate[62530]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-11T20:47:49.181 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:48 smithi204 podman[62515]: 2024-03-11 20:47:48.902597564 +0000 UTC m=+0.969388253 container died 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, org.label-schema.build-date=20240307, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-11T20:47:49.309 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:49 smithi083 ceph-mon[66365]: pgmap v253: 97 pgs: 3 active+undersized, 13 active+undersized+degraded, 8 peering, 4 stale+active+clean, 69 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 91/1629 objects degraded (5.586%) 2024-03-11T20:47:49.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:49 smithi204 ceph-mon[53271]: pgmap v253: 97 pgs: 3 active+undersized, 13 active+undersized+degraded, 8 peering, 4 stale+active+clean, 69 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 91/1629 objects degraded (5.586%) 2024-03-11T20:47:50.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 283404 9.79 MB/sec execute 573 sec latency 43.282 ms 2024-03-11T20:47:50.611 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 podman[62662]: 2024-03-11 20:47:50.331633649 +0000 UTC m=+1.422597363 container remove 634b63c27d156900ce17a97a7eac2b861bf65327dfb58949c9b713320806d2ff (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-11T20:47:50.611 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 podman[62718]: 2024-03-11 20:47:50.478506441 +0000 UTC m=+0.025777972 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:47:50.904 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:47:50.922 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 podman[62718]: 2024-03-11 20:47:50.616348435 +0000 UTC m=+0.163619952 container create 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, ceph=True) 2024-03-11T20:47:50.923 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 podman[62718]: 2024-03-11 20:47:50.827719505 +0000 UTC m=+0.374991030 container init 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, RELEASE=main-2b0101e, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307) 2024-03-11T20:47:50.923 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 podman[62718]: 2024-03-11 20:47:50.832435289 +0000 UTC m=+0.379706813 container start 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:47:50.923 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 ceph-osd[62736]: -- 172.21.15.204:0/208711972 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5576d890f680 con 0x5576d88d9000 2024-03-11T20:47:51.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 283687 9.79 MB/sec execute 574 sec latency 42.277 ms 2024-03-11T20:47:51.179 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 bash[62718]: 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 2024-03-11T20:47:51.179 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:50 smithi204 systemd[1]: Started Ceph osd.6 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:47:51.679 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:47:51.376+0000 7f4f26a3d7c0 -1 Falling back to public interface 2024-03-11T20:47:51.704 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (8m) 111s ago 18m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 111s ago 18m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (5m) 35s ago 17m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (8m) 111s ago 17m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (12m) 111s ago 12m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (12m) 35s ago 12m 17.7M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (7m) 111s ago 19m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (6m) 35s ago 17m 452M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 111s ago 19m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:47:51.705 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (6m) 35s ago 17m 44.7M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (11m) 111s ago 11m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (11m) 35s ago 11m 89.8M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 111s ago 17m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (8m) 35s ago 17m 18.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 111s ago 16m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 111s ago 16m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 111s ago 15m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 111s ago 15m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (78s) 35s ago 14m 265M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (40s) 35s ago 14m 12.7M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 starting - - - 2704M 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (13m) 35s ago 13m 648M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:47:51.706 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 111s ago 16m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:47:52.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 284013 9.79 MB/sec execute 575 sec latency 48.598 ms 2024-03-11T20:47:52.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:51 smithi083 ceph-mon[66365]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.6 MiB/s wr, 518 op/s; 192/1641 objects degraded (11.700%) 2024-03-11T20:47:52.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:51 smithi083 ceph-mon[66365]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:52.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:52.125 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:52.125 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:51 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:47:52.198 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 6 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 10 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:47:52.199 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:47:52.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-mon[53271]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.6 MiB/s wr, 518 op/s; 192/1641 objects degraded (11.700%) 2024-03-11T20:47:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-mon[53271]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:51 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:47:52.685 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "12/23 daemons upgraded", 2024-03-11T20:47:52.686 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:47:52.686 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:47:52.686 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:47:53.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 284377 9.79 MB/sec execute 576 sec latency 44.689 ms 2024-03-11T20:47:53.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:53 smithi083 ceph-mon[66365]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:53.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:53 smithi083 ceph-mon[66365]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:53.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:53 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/4119271352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:53.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:53 smithi204 ceph-mon[53271]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:53.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:53 smithi204 ceph-mon[53271]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:53.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:53 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/4119271352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:47:53.179 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 195/1695 objects degraded (11.504%), 25 pgs degraded 2024-03-11T20:47:53.179 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-11T20:47:53.179 INFO:teuthology.orchestra.run.smithi083.stdout: osd.6 (root=default,host=smithi204) is down 2024-03-11T20:47:53.179 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 195/1695 objects degraded (11.504%), 25 pgs degraded 2024-03-11T20:47:53.179 INFO:teuthology.orchestra.run.smithi083.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-03-11T20:47:53.180 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-03-11T20:47:53.181 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-03-11T20:47:54.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 284669 9.79 MB/sec execute 577 sec latency 50.460 ms 2024-03-11T20:47:54.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:54 smithi083 ceph-mon[66365]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:54.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:54 smithi083 ceph-mon[66365]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 195/1695 objects degraded (11.504%) 2024-03-11T20:47:54.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:54 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 195/1695 objects degraded (11.504%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:54.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:54 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/2125758177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:47:54.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:54 smithi204 ceph-mon[53271]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:47:54.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:54 smithi204 ceph-mon[53271]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 195/1695 objects degraded (11.504%) 2024-03-11T20:47:54.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:54 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 195/1695 objects degraded (11.504%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:54.160 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:54 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/2125758177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:47:55.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 284898 9.78 MB/sec execute 578 sec latency 64.550 ms 2024-03-11T20:47:56.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 285263 9.78 MB/sec execute 579 sec latency 66.423 ms 2024-03-11T20:47:56.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:56 smithi083 ceph-mon[66365]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.9 MiB/s wr, 547 op/s; 198/1722 objects degraded (11.498%) 2024-03-11T20:47:56.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:56.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:56.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:56 smithi204 ceph-mon[53271]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.9 MiB/s wr, 547 op/s; 198/1722 objects degraded (11.498%) 2024-03-11T20:47:56.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:56.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:57.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 285617 9.77 MB/sec execute 580 sec latency 50.783 ms 2024-03-11T20:47:57.993 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:57 smithi083 ceph-mon[66365]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 192/1653 objects degraded (11.615%) 2024-03-11T20:47:57.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:57.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:57.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:57.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:58.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 285925 9.77 MB/sec execute 581 sec latency 44.687 ms 2024-03-11T20:47:58.118 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:57 smithi204 ceph-mon[53271]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 192/1653 objects degraded (11.615%) 2024-03-11T20:47:58.118 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:58.118 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:47:58.118 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:58.118 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:47:59.012 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:58 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 195/1623 objects degraded (12.015%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:47:59.057 INFO:teuthology.orchestra.run.smithi083.stdout: 5 286348 9.79 MB/sec execute 582 sec latency 41.333 ms 2024-03-11T20:47:59.124 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:58 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 195/1623 objects degraded (12.015%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:00.028 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:47:59 smithi083 ceph-mon[66365]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 392 op/s; 195/1623 objects degraded (12.015%) 2024-03-11T20:48:00.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 286810 9.78 MB/sec execute 583 sec latency 35.549 ms 2024-03-11T20:48:00.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:47:59 smithi204 ceph-mon[53271]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 392 op/s; 195/1623 objects degraded (12.015%) 2024-03-11T20:48:00.430 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:48:00 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:48:00.117+0000 7f4f26a3d7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-03-11T20:48:01.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 287178 9.78 MB/sec execute 584 sec latency 39.201 ms 2024-03-11T20:48:01.929 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:48:01.538+0000 7f4f26a3d7c0 -1 osd.6 130 log_to_monitors true 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.6 MiB/s wr, 585 op/s; 195/1650 objects degraded (11.818%) 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:48:02.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:01 smithi083 ceph-mon[66365]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:48:02.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 287535 9.79 MB/sec execute 585 sec latency 48.556 ms 2024-03-11T20:48:02.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.6 MiB/s wr, 585 op/s; 195/1650 objects degraded (11.818%) 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:02.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:48:02.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:01 smithi204 ceph-mon[53271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-11T20:48:02.934 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:48:02.685+0000 7f4f13e8d700 -1 osd.6 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:48:03.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 287817 9.78 MB/sec execute 586 sec latency 116.734 ms 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: osdmap e133: 8 total, 7 up, 8 in 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: from='osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:03.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:02 smithi083 ceph-mon[66365]: from='osd.6 ' entity='osd.6' 2024-03-11T20:48:03.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:03.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-11T20:48:03.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-11T20:48:03.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: osdmap e133: 8 total, 7 up, 8 in 2024-03-11T20:48:03.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: from='osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:03.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:03.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:02 smithi204 ceph-mon[53271]: from='osd.6 ' entity='osd.6' 2024-03-11T20:48:04.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 288132 9.78 MB/sec execute 587 sec latency 360.213 ms 2024-03-11T20:48:04.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:03 smithi083 ceph-mon[66365]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.1 MiB/s wr, 546 op/s; 196/1689 objects degraded (11.604%) 2024-03-11T20:48:04.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:03 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:48:04.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:03 smithi083 ceph-mon[66365]: osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410] boot 2024-03-11T20:48:04.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:03 smithi083 ceph-mon[66365]: osdmap e134: 8 total, 8 up, 8 in 2024-03-11T20:48:04.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:48:04.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:03 smithi204 ceph-mon[53271]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.1 MiB/s wr, 546 op/s; 196/1689 objects degraded (11.604%) 2024-03-11T20:48:04.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:03 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:48:04.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:03 smithi204 ceph-mon[53271]: osd.6 [v2:172.21.15.204:6816/104309410,v1:172.21.15.204:6817/104309410] boot 2024-03-11T20:48:04.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:03 smithi204 ceph-mon[53271]: osdmap e134: 8 total, 8 up, 8 in 2024-03-11T20:48:04.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-11T20:48:05.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 288132 9.77 MB/sec execute 588 sec latency 1360.267 ms 2024-03-11T20:48:05.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:05 smithi083 ceph-mon[66365]: osdmap e135: 8 total, 8 up, 8 in 2024-03-11T20:48:05.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:05 smithi083 ceph-mon[66365]: pgmap v264: 97 pgs: 7 peering, 5 active+undersized, 23 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 533 op/s; 170/1662 objects degraded (10.229%) 2024-03-11T20:48:05.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:05 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 196/1689 objects degraded (11.604%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:06.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 288208 9.75 MB/sec execute 589 sec latency 2046.035 ms 2024-03-11T20:48:06.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:05 smithi204 ceph-mon[53271]: osdmap e135: 8 total, 8 up, 8 in 2024-03-11T20:48:06.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:05 smithi204 ceph-mon[53271]: pgmap v264: 97 pgs: 7 peering, 5 active+undersized, 23 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 533 op/s; 170/1662 objects degraded (10.229%) 2024-03-11T20:48:06.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:05 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 196/1689 objects degraded (11.604%), 25 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:06.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:06 smithi083 ceph-mon[66365]: osdmap e136: 8 total, 8 up, 8 in 2024-03-11T20:48:07.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 288650 9.75 MB/sec execute 590 sec latency 336.412 ms 2024-03-11T20:48:07.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:06 smithi204 ceph-mon[53271]: osdmap e136: 8 total, 8 up, 8 in 2024-03-11T20:48:07.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:07 smithi083 ceph-mon[66365]: osdmap e137: 8 total, 8 up, 8 in 2024-03-11T20:48:07.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:07 smithi083 ceph-mon[66365]: pgmap v267: 97 pgs: 5 remapped+peering, 7 peering, 2 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 67 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 118/1653 objects degraded (7.139%); 0 B/s, 72 keys/s, 1 objects/s recovering 2024-03-11T20:48:08.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 288913 9.75 MB/sec execute 591 sec latency 994.961 ms 2024-03-11T20:48:08.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:07 smithi204 ceph-mon[53271]: osdmap e137: 8 total, 8 up, 8 in 2024-03-11T20:48:08.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:07 smithi204 ceph-mon[53271]: pgmap v267: 97 pgs: 5 remapped+peering, 7 peering, 2 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized+degraded, 67 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 118/1653 objects degraded (7.139%); 0 B/s, 72 keys/s, 1 objects/s recovering 2024-03-11T20:48:08.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:08 smithi083 ceph-mon[66365]: osdmap e138: 8 total, 8 up, 8 in 2024-03-11T20:48:09.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 289307 9.75 MB/sec execute 592 sec latency 610.588 ms 2024-03-11T20:48:09.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:08 smithi204 ceph-mon[53271]: osdmap e138: 8 total, 8 up, 8 in 2024-03-11T20:48:09.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:09 smithi083 ceph-mon[66365]: osdmap e139: 8 total, 8 up, 8 in 2024-03-11T20:48:09.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:09 smithi083 ceph-mon[66365]: pgmap v270: 97 pgs: 3 remapped+peering, 12 peering, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.9 MiB/s wr, 658 op/s; 34/1620 objects degraded (2.099%); 1.1 MiB/s, 72 keys/s, 30 objects/s recovering 2024-03-11T20:48:10.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 289759 9.74 MB/sec execute 593 sec latency 63.056 ms 2024-03-11T20:48:10.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:09 smithi204 ceph-mon[53271]: osdmap e139: 8 total, 8 up, 8 in 2024-03-11T20:48:10.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:09 smithi204 ceph-mon[53271]: pgmap v270: 97 pgs: 3 remapped+peering, 12 peering, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.9 MiB/s wr, 658 op/s; 34/1620 objects degraded (2.099%); 1.1 MiB/s, 72 keys/s, 30 objects/s recovering 2024-03-11T20:48:10.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:10 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 34/1620 objects degraded (2.099%), 2 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:11.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 290034 9.74 MB/sec execute 594 sec latency 43.373 ms 2024-03-11T20:48:11.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:10 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 34/1620 objects degraded (2.099%), 2 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:11.818 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:11 smithi083 ceph-mon[66365]: pgmap v271: 97 pgs: 5 peering, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 34/1626 objects degraded (2.091%); 2.1 MiB/s, 55 keys/s, 41 objects/s recovering 2024-03-11T20:48:12.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 290335 9.75 MB/sec execute 595 sec latency 48.466 ms 2024-03-11T20:48:12.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:11 smithi204 ceph-mon[53271]: pgmap v271: 97 pgs: 5 peering, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 34/1626 objects degraded (2.091%); 2.1 MiB/s, 55 keys/s, 41 objects/s recovering 2024-03-11T20:48:13.059 INFO:teuthology.orchestra.run.smithi083.stdout: 5 290679 9.75 MB/sec execute 596 sec latency 46.703 ms 2024-03-11T20:48:13.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:13.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:13.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:13.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:13.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:13 smithi083 ceph-mon[66365]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 469 op/s; 3.3 MiB/s, 39 objects/s recovering 2024-03-11T20:48:13.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:13 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1626 objects degraded (2.091%), 2 pgs degraded) 2024-03-11T20:48:13.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:13 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:48:14.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 291051 9.76 MB/sec execute 597 sec latency 55.233 ms 2024-03-11T20:48:14.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:13 smithi204 ceph-mon[53271]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 469 op/s; 3.3 MiB/s, 39 objects/s recovering 2024-03-11T20:48:14.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:13 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1626 objects degraded (2.091%), 2 pgs degraded) 2024-03-11T20:48:14.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:13 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:48:15.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 291311 9.75 MB/sec execute 598 sec latency 43.955 ms 2024-03-11T20:48:15.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:15 smithi083 ceph-mon[66365]: pgmap v273: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 427 op/s; 2.5 MiB/s, 30 objects/s recovering 2024-03-11T20:48:16.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 291631 9.75 MB/sec execute 599 sec latency 50.762 ms 2024-03-11T20:48:16.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:15 smithi204 ceph-mon[53271]: pgmap v273: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 427 op/s; 2.5 MiB/s, 30 objects/s recovering 2024-03-11T20:48:16.969 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:16 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:17.058 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 600 sec 2024-03-11T20:48:17.149 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:16 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: 0 cleanup 600 sec 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: Operation Count AvgLat MaxLat 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: ---------------------------------------- 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: NTCreateX 187560 4.590 2630.047 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: Close 137689 0.133 6.612 2024-03-11T20:48:17.327 INFO:teuthology.orchestra.run.smithi083.stdout: Rename 7949 9.692 1051.146 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Unlink 37967 7.302 2160.728 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Qpathinfo 170105 1.132 1065.682 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Qfileinfo 29639 0.008 2.460 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Qfsinfo 31197 0.589 86.026 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Sfileinfo 15312 13.716 2338.524 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Find 65723 0.933 23.030 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: WriteX 92671 13.785 2646.585 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: ReadX 293994 0.010 5.153 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: LockX 610 0.373 5.557 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: UnlockX 610 0.313 1.720 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: Flush 13190 0.002 0.156 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:48:17.328 INFO:teuthology.orchestra.run.smithi083.stdout:Throughput 9.74714 MB/sec 5 clients 5 procs max_latency=2646.593 ms 2024-03-11T20:48:17.330 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-03-11T20:48:17.390 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-03-11T20:48:17.490 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-03-11T20:48:17.516 INFO:teuthology.orchestra.run.smithi083.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-11T20:48:17.516 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:48:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-03-11T20:48:17.517 INFO:teuthology.orchestra.run.smithi083.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-11T20:48:17.617 INFO:teuthology.orchestra.run.smithi083.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-11T20:48:17.617 INFO:teuthology.orchestra.run.smithi083.stdout:releasing clients 2024-03-11T20:48:18.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:17 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:18.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:17 smithi083 ceph-mon[66365]: Upgrade: osd.7 is safe to restart 2024-03-11T20:48:18.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:17 smithi083 ceph-mon[66365]: pgmap v274: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 411 op/s; 1.7 MiB/s, 23 objects/s recovering 2024-03-11T20:48:18.153 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:17 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-11T20:48:18.154 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:17 smithi204 ceph-mon[53271]: Upgrade: osd.7 is safe to restart 2024-03-11T20:48:18.154 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:17 smithi204 ceph-mon[53271]: pgmap v274: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 411 op/s; 1.7 MiB/s, 23 objects/s recovering 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: Upgrade: Updating osd.7 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: Deploying daemon osd.7 on smithi204 2024-03-11T20:48:19.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:19 smithi083 ceph-mon[66365]: pgmap v275: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.5 MiB/s wr, 449 op/s; 1.6 MiB/s, 12 objects/s recovering 2024-03-11T20:48:19.358 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: Upgrade: Updating osd.7 2024-03-11T20:48:19.359 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:19.359 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-11T20:48:19.359 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:19.359 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: Deploying daemon osd.7 on smithi204 2024-03-11T20:48:19.359 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:19 smithi204 ceph-mon[53271]: pgmap v275: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.5 MiB/s wr, 449 op/s; 1.6 MiB/s, 12 objects/s recovering 2024-03-11T20:48:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261 17.17 MB/sec execute 1 sec latency 362.044 ms 2024-03-11T20:48:20.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 432 22.11 MB/sec execute 2 sec latency 35.813 ms 2024-03-11T20:48:21.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 658 24.51 MB/sec execute 3 sec latency 28.982 ms 2024-03-11T20:48:21.959 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:21 smithi083 ceph-mon[66365]: pgmap v276: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.7 MiB/s wr, 416 op/s; 1.3 MiB/s, 10 objects/s recovering 2024-03-11T20:48:22.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:21 smithi204 ceph-mon[53271]: pgmap v276: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.7 MiB/s wr, 416 op/s; 1.3 MiB/s, 10 objects/s recovering 2024-03-11T20:48:22.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 918 19.21 MB/sec execute 4 sec latency 42.970 ms 2024-03-11T20:48:22.657 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:22 smithi204 systemd[1]: Stopping Ceph osd.7 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:48:22.929 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[40505]: 2024-03-11T20:48:22.698+0000 7f6b52aee700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:48:22.930 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[40505]: 2024-03-11T20:48:22.698+0000 7f6b52aee700 -1 osd.7 139 *** Got signal Terminated *** 2024-03-11T20:48:22.930 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[40505]: 2024-03-11T20:48:22.698+0000 7f6b52aee700 -1 osd.7 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:48:23.227 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:23 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:23.227 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:23 smithi083 ceph-mon[66365]: osd.7 marked itself down and dead 2024-03-11T20:48:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:23 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:23.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:23 smithi204 ceph-mon[53271]: osd.7 marked itself down and dead 2024-03-11T20:48:23.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 5 sec 2024-03-11T20:48:23.625 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:48:24.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:23 smithi204 podman[64538]: 2024-03-11 20:48:23.908803711 +0000 UTC m=+1.350946762 container died 8fdc41d49d8846d537641a87bbfd908097e732d1686e334ca650c67f5183a21c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, version=8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754, vcs-type=git, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (9m) 2m ago 18m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 2m ago 18m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (5m) 28s ago 18m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (9m) 2m ago 18m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:48:24.374 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (12m) 2m ago 12m 39.2M - 17.2.0 e1d6a67b021e 38b421853ee3 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (12m) 28s ago 12m 17.9M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (8m) 2m ago 19m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (7m) 28s ago 17m 452M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (7m) 2m ago 20m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (6m) 28s ago 17m 46.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (12m) 2m ago 12m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (12m) 28s ago 12m 89.8M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (9m) 2m ago 18m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (8m) 28s ago 17m 15.2M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 2m ago 17m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 2m ago 16m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 16m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 15m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (111s) 28s ago 15m 310M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:48:24.375 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (72s) 28s ago 14m 261M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:48:24.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (33s) 28s ago 14m 11.7M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:48:24.376 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (14m) 28s ago 14m 703M 2704M 17.2.0 e1d6a67b021e 8fdc41d49d88 2024-03-11T20:48:24.376 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 2m ago 17m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:48:24.432 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:24 smithi083 ceph-mon[66365]: pgmap v277: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 9.3 MiB/s wr, 511 op/s; 770 KiB/s, 2 objects/s recovering 2024-03-11T20:48:24.432 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:24 smithi083 ceph-mon[66365]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:48:24.432 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:24 smithi083 ceph-mon[66365]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-11T20:48:24.432 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:24 smithi083 ceph-mon[66365]: osdmap e140: 8 total, 7 up, 8 in 2024-03-11T20:48:24.438 INFO:teuthology.orchestra.run.smithi083.stdout: 0 cleanup 6 sec 2024-03-11T20:48:24.438 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Operation Count AvgLat MaxLat 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: ---------------------------------------- 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: NTCreateX 780 8.396 62.432 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Close 721 9.371 1021.753 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Rename 41 14.038 42.960 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Unlink 74 3.088 19.755 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Qpathinfo 399 1.317 23.530 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Qfileinfo 376 25.458 1024.428 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Qfsinfo 148 0.463 3.485 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Sfileinfo 142 12.589 25.917 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: Find 200 0.720 3.273 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: WriteX 1804 0.025 0.070 2024-03-11T20:48:24.439 INFO:teuthology.orchestra.run.smithi083.stdout: ReadX 810 0.017 0.362 2024-03-11T20:48:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: Flush 120 14.422 29.031 2024-03-11T20:48:24.440 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:48:24.440 INFO:teuthology.orchestra.run.smithi083.stdout:Throughput 19.2069 MB/sec 5 clients 5 procs max_latency=362.044 ms 2024-03-11T20:48:24.441 DEBUG:teuthology.parallel:result is None 2024-03-11T20:48:24.562 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:24 smithi204 ceph-mon[53271]: pgmap v277: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 9.3 MiB/s wr, 511 op/s; 770 KiB/s, 2 objects/s recovering 2024-03-11T20:48:24.562 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:24 smithi204 ceph-mon[53271]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-11T20:48:24.562 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:24 smithi204 ceph-mon[53271]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-11T20:48:24.562 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:24 smithi204 ceph-mon[53271]: osdmap e140: 8 total, 7 up, 8 in 2024-03-11T20:48:24.563 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:24 smithi204 podman[64538]: 2024-03-11 20:48:24.324207096 +0000 UTC m=+1.766350129 container cleanup 8fdc41d49d8846d537641a87bbfd908097e732d1686e334ca650c67f5183a21c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vcs-type=git, version=8, GIT_CLEAN=True) 2024-03-11T20:48:24.563 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:24 smithi204 bash[64538]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:48:24.800 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 7 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 11 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:48:24.801 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:48:24.929 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:24 smithi204 podman[64554]: 2024-03-11 20:48:24.567542525 +0000 UTC m=+0.655303664 container remove 8fdc41d49d8846d537641a87bbfd908097e732d1686e334ca650c67f5183a21c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., ceph=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: "crash" 2024-03-11T20:48:25.151 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:48:25.152 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "13/23 daemons upgraded", 2024-03-11T20:48:25.152 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-03-11T20:48:25.152 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:48:25.152 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:48:25.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:25 smithi204 ceph-mon[53271]: from='client.44345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:25 smithi204 ceph-mon[53271]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:25 smithi204 ceph-mon[53271]: osdmap e141: 8 total, 7 up, 8 in 2024-03-11T20:48:25.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:25 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3770102309' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:25.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:24 smithi204 podman[64647]: 2024-03-11 20:48:24.898130073 +0000 UTC m=+0.018119502 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:48:25.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:25 smithi204 podman[64647]: 2024-03-11 20:48:25.018625608 +0000 UTC m=+0.138615057 container create 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307) 2024-03-11T20:48:25.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:25 smithi204 podman[64647]: 2024-03-11 20:48:25.2350074 +0000 UTC m=+0.354996820 container init 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=main-2b0101e, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307) 2024-03-11T20:48:25.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:25 smithi204 podman[64647]: 2024-03-11 20:48:25.241081932 +0000 UTC m=+0.361071339 container start 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux ) 2024-03-11T20:48:25.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:25 smithi083 ceph-mon[66365]: from='client.44345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:25.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:25 smithi083 ceph-mon[66365]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:25.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:25 smithi083 ceph-mon[66365]: osdmap e141: 8 total, 7 up, 8 in 2024-03-11T20:48:25.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:25 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3770102309' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:25.568 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Reduced data availability: 2 pgs peering 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout: osd.7 (root=default,host=smithi204) is down 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs peering 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.b is stuck peering for 2m, current state peering, last acting [4,5] 2024-03-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1c is stuck peering for 81s, current state peering, last acting [5,2] 2024-03-11T20:48:25.912 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:25 smithi204 podman[64647]: 2024-03-11 20:48:25.471283305 +0000 UTC m=+0.591272736 container attach 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307) 2024-03-11T20:48:25.912 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:25 smithi204 podman[64647]: 2024-03-11 20:48:25.556387203 +0000 UTC m=+0.676376613 container died 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-2b0101e, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-11T20:48:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:26 smithi083 ceph-mon[66365]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:26 smithi083 ceph-mon[66365]: pgmap v280: 97 pgs: 16 peering, 5 stale+active+clean, 76 active+clean; 317 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 530 op/s 2024-03-11T20:48:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:26 smithi083 ceph-mon[66365]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:26 smithi083 ceph-mon[66365]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-11T20:48:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:26 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3791032618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:48:26.491 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:26 smithi204 ceph-mon[53271]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:26.491 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:26 smithi204 ceph-mon[53271]: pgmap v280: 97 pgs: 16 peering, 5 stale+active+clean, 76 active+clean; 317 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 530 op/s 2024-03-11T20:48:26.491 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:26 smithi204 ceph-mon[53271]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:26.491 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:26 smithi204 ceph-mon[53271]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-11T20:48:26.491 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:26 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3791032618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:48:26.491 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 podman[64673]: 2024-03-11 20:48:26.20854251 +0000 UTC m=+0.647220575 container remove 0b62ed7c18149663d6809895568369f2e6d1dbd8329de6d3e92d5e3ee9d2923a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-11T20:48:26.491 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.7.service: Deactivated successfully. 2024-03-11T20:48:26.491 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 systemd[1]: Stopped Ceph osd.7 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:48:26.491 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.7.service: Consumed 8min 3.053s CPU time. 2024-03-11T20:48:26.835 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 systemd[1]: Starting Ceph osd.7 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:48:26.835 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 podman[64775]: 2024-03-11 20:48:26.735128644 +0000 UTC m=+0.111870192 container create c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-11T20:48:26.835 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 podman[64775]: 2024-03-11 20:48:26.641493828 +0000 UTC m=+0.018235357 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:48:27.179 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 podman[64775]: 2024-03-11 20:48:26.962892958 +0000 UTC m=+0.339634463 container init c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-11T20:48:27.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:26 smithi204 podman[64775]: 2024-03-11 20:48:26.966025559 +0000 UTC m=+0.342767069 container start c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:48:27.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 podman[64775]: 2024-03-11 20:48:27.05448026 +0000 UTC m=+0.431221781 container attach c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True) 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-11T20:48:27.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 bash[64775]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate[64790]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-11T20:48:27.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:27 smithi204 podman[64775]: 2024-03-11 20:48:27.432268864 +0000 UTC m=+0.809010369 container died c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, GIT_CLEAN=True, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240307) 2024-03-11T20:48:28.054 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-mon[53271]: pgmap v281: 97 pgs: 5 active+undersized, 16 active+undersized+degraded, 16 peering, 1 stale+active+clean, 59 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s wr, 350 op/s; 88/1401 objects degraded (6.281%) 2024-03-11T20:48:28.054 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:28.054 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:28.054 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:27 smithi204 ceph-mon[53271]: Health check failed: Degraded data redundancy: 88/1401 objects degraded (6.281%), 16 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:28.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:27 smithi083 ceph-mon[66365]: pgmap v281: 97 pgs: 5 active+undersized, 16 active+undersized+degraded, 16 peering, 1 stale+active+clean, 59 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s wr, 350 op/s; 88/1401 objects degraded (6.281%) 2024-03-11T20:48:28.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:28.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:28.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:27 smithi083 ceph-mon[66365]: Health check failed: Degraded data redundancy: 88/1401 objects degraded (6.281%), 16 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:28.429 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:28 smithi204 podman[64906]: 2024-03-11 20:48:28.058897182 +0000 UTC m=+0.615647709 container remove c8d9a213f4b33f23c2c67979d6028f6b1e611544892599d80d5ca364487710f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-11T20:48:28.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:28 smithi204 podman[64971]: 2024-03-11 20:48:28.175101492 +0000 UTC m=+0.017613093 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:48:28.430 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:28 smithi204 podman[64971]: 2024-03-11 20:48:28.293008587 +0000 UTC m=+0.135520175 container create a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, GIT_BRANCH=HEAD) 2024-03-11T20:48:30.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:29 smithi204 ceph-mon[53271]: pgmap v282: 97 pgs: 7 active+undersized, 27 active+undersized+degraded, 8 peering, 55 active+clean; 317 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 361 op/s; 122/1200 objects degraded (10.167%) 2024-03-11T20:48:30.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:29 smithi204 podman[64971]: 2024-03-11 20:48:29.780269117 +0000 UTC m=+1.622780704 container init a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=main-2b0101e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:48:30.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:29 smithi204 podman[64971]: 2024-03-11 20:48:29.784120204 +0000 UTC m=+1.626631793 container start a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-11T20:48:30.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:29 smithi204 ceph-osd[64991]: -- 172.21.15.204:0/3488799748 <== mon.1 v2:172.21.15.204:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e7de2e9680 con 0x55e7de2b3000 2024-03-11T20:48:30.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:29 smithi204 bash[64971]: a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 2024-03-11T20:48:30.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:29 smithi204 systemd[1]: Started Ceph osd.7 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:48:30.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:29 smithi083 ceph-mon[66365]: pgmap v282: 97 pgs: 7 active+undersized, 27 active+undersized+degraded, 8 peering, 55 active+clean; 317 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 361 op/s; 122/1200 objects degraded (10.167%) 2024-03-11T20:48:30.679 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:48:30.331+0000 7fed8eb3c7c0 -1 Falling back to public interface 2024-03-11T20:48:31.019 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:30 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:31.019 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:30 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:31.019 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:30 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:31.019 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:30 smithi204 ceph-mon[53271]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:48:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:30 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:30 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:30 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:30 smithi083 ceph-mon[66365]: Health check update: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:48:32.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:31 smithi204 ceph-mon[53271]: pgmap v283: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 249 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s wr, 173 op/s; 88/567 objects degraded (15.520%) 2024-03-11T20:48:32.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:31 smithi204 ceph-mon[53271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-11T20:48:32.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:31 smithi083 ceph-mon[66365]: pgmap v283: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 249 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s wr, 173 op/s; 88/567 objects degraded (15.520%) 2024-03-11T20:48:32.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:31 smithi083 ceph-mon[66365]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-11T20:48:33.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:32 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 88/450 objects degraded (19.556%), 32 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:33.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:32 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 88/450 objects degraded (19.556%), 32 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:34.210 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:33 smithi204 ceph-mon[53271]: pgmap v284: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 236 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 147 op/s; 88/450 objects degraded (19.556%) 2024-03-11T20:48:34.210 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:34.210 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:33 smithi083 ceph-mon[66365]: pgmap v284: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 236 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 147 op/s; 88/450 objects degraded (19.556%) 2024-03-11T20:48:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:36.155 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:36.155 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:35 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:36.155 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:35 smithi204 ceph-mon[53271]: pgmap v285: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 107 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:36.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:36.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:35 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:36.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:35 smithi083 ceph-mon[66365]: pgmap v285: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 107 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:38.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:37 smithi204 ceph-mon[53271]: pgmap v286: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s wr, 95 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:37 smithi083 ceph-mon[66365]: pgmap v286: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s wr, 95 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:38.929 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:48:38.561+0000 7fed8eb3c7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-03-11T20:48:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all osd 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-11T20:48:39.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: Upgrade: Setting require_osd_release to 19 squid 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-11T20:48:39.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:39 smithi204 ceph-mon[53271]: pgmap v287: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 248 KiB/s wr, 67 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:39.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all osd 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-11T20:48:39.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-11T20:48:39.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-11T20:48:39.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-11T20:48:39.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: Upgrade: Setting require_osd_release to 19 squid 2024-03-11T20:48:39.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-11T20:48:39.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:39 smithi083 ceph-mon[66365]: pgmap v287: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 248 KiB/s wr, 67 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:40.429 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:48:40.088+0000 7fed8eb3c7c0 -1 osd.7 139 log_to_monitors true 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: osdmap e142: 8 total, 7 up, 8 in 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi083.motgne"]}]: dispatch 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: Upgrade: It appears safe to stop mds.foofs.smithi083.motgne 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: from='osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:48:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-mon[53271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: osdmap e142: 8 total, 7 up, 8 in 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi083.motgne"]}]: dispatch 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: Upgrade: It appears safe to stop mds.foofs.smithi083.motgne 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: from='osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:48:40.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:40 smithi083 ceph-mon[66365]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-11T20:48:41.179 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:48:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:48:40.899+0000 7fed7bf8c700 -1 osd.7 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: osdmap e143: 8 total, 7 up, 8 in 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 85/432 objects degraded (19.676%), 32 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:41.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: pgmap v290: 97 pgs: 12 active+undersized, 30 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 23 KiB/s wr, 1 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:41.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='osd.7 ' entity='osd.7' 2024-03-11T20:48:41.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:41.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:48:41.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: osdmap e143: 8 total, 7 up, 8 in 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi204", "root=default"]}]: dispatch 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 85/432 objects degraded (19.676%), 32 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: pgmap v290: 97 pgs: 12 active+undersized, 30 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 23 KiB/s wr, 1 op/s; 85/432 objects degraded (19.676%) 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='osd.7 ' entity='osd.7' 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:41.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.motgne", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:48:41.931 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: Upgrade: Updating mds.foofs.smithi083.motgne 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: Deploying daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487] boot 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: osdmap e144: 8 total, 8 up, 8 in 2024-03-11T20:48:42.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: Upgrade: Updating mds.foofs.smithi083.motgne 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: Deploying daemon mds.foofs.smithi083.motgne on smithi083 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: osd.7 [v2:172.21.15.204:6824/1276092487,v1:172.21.15.204:6825/1276092487] boot 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: osdmap e144: 8 total, 8 up, 8 in 2024-03-11T20:48:42.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-11T20:48:44.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:43 smithi204 ceph-mon[53271]: pgmap v292: 97 pgs: 4 peering, 12 active+undersized, 26 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s; 72/432 objects degraded (16.667%) 2024-03-11T20:48:44.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:43 smithi204 ceph-mon[53271]: osdmap e145: 8 total, 8 up, 8 in 2024-03-11T20:48:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:43 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[66361]: 2024-03-11T20:48:43.898+0000 7fb104963700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:48:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:43 smithi083 ceph-mon[66365]: pgmap v292: 97 pgs: 4 peering, 12 active+undersized, 26 active+undersized+degraded, 55 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s; 72/432 objects degraded (16.667%) 2024-03-11T20:48:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:43 smithi083 ceph-mon[66365]: osdmap e145: 8 total, 8 up, 8 in 2024-03-11T20:48:44.197 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:48:43 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:48:43.990+0000 7fe5147f1700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 13 2024-03-11T20:48:44.197 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:48:43 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:48:43.990+0000 7fe515ff4700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 12 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: osdmap e146: 8 total, 8 up, 8 in 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: Standby daemon mds.foofs.smithi204.evhral assigned to filesystem foofs as rank 0 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-11T20:48:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:44 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:replay} 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: osdmap e146: 8 total, 8 up, 8 in 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: Standby daemon mds.foofs.smithi204.evhral assigned to filesystem foofs as rank 0 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-11T20:48:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:44 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:replay} 2024-03-11T20:48:46.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:45 smithi204 ceph-mon[53271]: pgmap v295: 97 pgs: 4 peering, 10 active+undersized, 19 active+undersized+degraded, 64 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 230 B/s rd, 0 op/s; 61/432 objects degraded (14.120%); 0 B/s, 7 objects/s recovering 2024-03-11T20:48:46.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:45 smithi204 ceph-mon[53271]: Health check update: Degraded data redundancy: 61/432 objects degraded (14.120%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:46.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:45 smithi083 ceph-mon[66365]: pgmap v295: 97 pgs: 4 peering, 10 active+undersized, 19 active+undersized+degraded, 64 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 230 B/s rd, 0 op/s; 61/432 objects degraded (14.120%); 0 B/s, 7 objects/s recovering 2024-03-11T20:48:46.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:45 smithi083 ceph-mon[66365]: Health check update: Degraded data redundancy: 61/432 objects degraded (14.120%), 19 pgs degraded (PG_DEGRADED) 2024-03-11T20:48:47.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:46 smithi204 ceph-mon[53271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 61/432 objects degraded (14.120%), 19 pgs degraded) 2024-03-11T20:48:47.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:46 smithi083 ceph-mon[66365]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 61/432 objects degraded (14.120%), 19 pgs degraded) 2024-03-11T20:48:48.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:47 smithi204 ceph-mon[53271]: pgmap v296: 97 pgs: 4 peering, 93 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5 op/s; 289 KiB/s, 22 objects/s recovering 2024-03-11T20:48:48.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:47 smithi083 ceph-mon[66365]: pgmap v296: 97 pgs: 4 peering, 93 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5 op/s; 289 KiB/s, 22 objects/s recovering 2024-03-11T20:48:49.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:48 smithi204 ceph-mon[53271]: reconnect by client.24357 172.21.15.204:0/1037687027 after 0 2024-03-11T20:48:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:48 smithi204 ceph-mon[53271]: reconnect by client.14554 172.21.15.83:0/914133764 after 0 2024-03-11T20:48:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:48 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:reconnect 2024-03-11T20:48:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:48 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:reconnect} 2024-03-11T20:48:49.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:48 smithi083 ceph-mon[66365]: reconnect by client.24357 172.21.15.204:0/1037687027 after 0 2024-03-11T20:48:49.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:48 smithi083 ceph-mon[66365]: reconnect by client.14554 172.21.15.83:0/914133764 after 0 2024-03-11T20:48:49.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:48 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:reconnect 2024-03-11T20:48:49.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:48 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:reconnect} 2024-03-11T20:48:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:49 smithi204 ceph-mon[53271]: pgmap v297: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 7 op/s; 248 KiB/s, 19 objects/s recovering 2024-03-11T20:48:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:49 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:rejoin 2024-03-11T20:48:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:49 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:rejoin} 2024-03-11T20:48:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:49 smithi204 ceph-mon[53271]: daemon mds.foofs.smithi204.evhral is now active in filesystem foofs as rank 0 2024-03-11T20:48:50.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:49 smithi083 ceph-mon[66365]: pgmap v297: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 7 op/s; 248 KiB/s, 19 objects/s recovering 2024-03-11T20:48:50.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:49 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:rejoin 2024-03-11T20:48:50.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:49 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi204.evhral=up:rejoin} 2024-03-11T20:48:50.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:49 smithi083 ceph-mon[66365]: daemon mds.foofs.smithi204.evhral is now active in filesystem foofs as rank 0 2024-03-11T20:48:51.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:50 smithi204 ceph-mon[53271]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-11T20:48:51.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:50 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:active 2024-03-11T20:48:51.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:50 smithi204 ceph-mon[53271]: fsmap foofs:1 {0=foofs.smithi204.evhral=up:active} 2024-03-11T20:48:51.446 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:50 smithi083 ceph-mon[66365]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-11T20:48:51.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:50 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.204:6832/1390110615,v1:172.21.15.204:6833/1390110615] up:active 2024-03-11T20:48:51.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:50 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi204.evhral=up:active} 2024-03-11T20:48:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:51 smithi204 ceph-mon[53271]: pgmap v298: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 127 B/s wr, 12 op/s; 217 KiB/s, 17 objects/s recovering 2024-03-11T20:48:52.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:51 smithi083 ceph-mon[66365]: pgmap v298: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 127 B/s wr, 12 op/s; 217 KiB/s, 17 objects/s recovering 2024-03-11T20:48:54.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:53 smithi083 ceph-mon[66365]: pgmap v299: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 205 B/s wr, 11 op/s; 174 KiB/s, 13 objects/s recovering 2024-03-11T20:48:54.372 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:53 smithi204 ceph-mon[53271]: pgmap v299: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 205 B/s wr, 11 op/s; 174 KiB/s, 13 objects/s recovering 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: pgmap v300: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 186 B/s wr, 10 op/s; 158 KiB/s, 11 objects/s recovering 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-11T20:48:55.927 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:48:55.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:boot 2024-03-11T20:48:55.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi204.evhral=up:active} 1 up:standby 2024-03-11T20:48:55.928 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:48:55.928 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:48:56.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:56.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:56.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:48:56.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: pgmap v300: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 186 B/s wr, 10 op/s; 158 KiB/s, 11 objects/s recovering 2024-03-11T20:48:56.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-11T20:48:56.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:48:56.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:boot 2024-03-11T20:48:56.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: fsmap foofs:1 {0=foofs.smithi204.evhral=up:active} 1 up:standby 2024-03-11T20:48:56.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (10m) 2m ago 19m 16.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 2m ago 19m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (6m) 22s ago 18m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (9m) 2m ago 18m 45.3M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 starting - - - - 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (13m) 22s ago 13m 18.0M - 17.2.0 e1d6a67b021e fdfae4b90ade 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (8m) 2m ago 20m 508M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:48:56.625 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (8m) 22s ago 18m 452M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (7m) 2m ago 20m 51.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (7m) 22s ago 18m 48.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (12m) 2m ago 12m 284M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (12m) 22s ago 12m 89.8M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (9m) 2m ago 18m 11.5M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (9m) 22s ago 18m 15.1M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 2m ago 17m 335M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 2m ago 17m 289M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 16m 222M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 2m ago 16m 15.0M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (2m) 22s ago 15m 345M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:48:56.626 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (105s) 22s ago 15m 312M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:48:56.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (66s) 22s ago 15m 286M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:48:56.627 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (28s) 22s ago 14m 11.8M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:48:56.627 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 2m ago 17m 70.2M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:48:56.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:56 smithi083 ceph-mon[66365]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:48:57.055 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 1 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 13 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:48:57.056 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:48:57.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:56 smithi204 ceph-mon[53271]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:57.399 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:48:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:48:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:48:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:48:57.399 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "osd" 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "14/23 daemons upgraded", 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mds daemons", 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:48:57.400 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:48:57.811 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: pgmap v301: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 144 KiB/s, 8 objects/s recovering 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/915759121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:58.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:57 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/637093946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: pgmap v301: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 144 KiB/s, 8 objects/s recovering 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/915759121' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:48:58.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:57 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/637093946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:48:59.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:58 smithi204 ceph-mon[53271]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:48:59.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:58 smithi083 ceph-mon[66365]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:00.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:48:59 smithi204 ceph-mon[53271]: pgmap v302: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-03-11T20:49:00.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:48:59 smithi083 ceph-mon[66365]: pgmap v302: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-03-11T20:49:01.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:01.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:01.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:01.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:02 smithi083 ceph-mon[66365]: pgmap v303: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-03-11T20:49:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:02 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:02.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:02 smithi204 ceph-mon[53271]: pgmap v303: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-03-11T20:49:02.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:02.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:02 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:04.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:04 smithi083 ceph-mon[66365]: pgmap v304: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 7.1 KiB/s wr, 4 op/s 2024-03-11T20:49:04.444 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:04 smithi204 ceph-mon[53271]: pgmap v304: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 7.1 KiB/s wr, 4 op/s 2024-03-11T20:49:06.300 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: pgmap v305: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.0 KiB/s wr, 3 op/s 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.301 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi204.evhral"]}]: dispatch 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: pgmap v305: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.0 KiB/s wr, 3 op/s 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:06.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:06.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi204.evhral"]}]: dispatch 2024-03-11T20:49:07.429 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:07 smithi204 ceph-mon[53271]: Detected new or changed devices on smithi083 2024-03-11T20:49:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:07 smithi204 ceph-mon[53271]: Upgrade: It appears safe to stop mds.foofs.smithi204.evhral 2024-03-11T20:49:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:49:07.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:07.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:07 smithi083 ceph-mon[66365]: Detected new or changed devices on smithi083 2024-03-11T20:49:07.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:07 smithi083 ceph-mon[66365]: Upgrade: It appears safe to stop mds.foofs.smithi204.evhral 2024-03-11T20:49:07.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:07.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi204.evhral", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-11T20:49:07.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:08.402 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:08 smithi204 ceph-mon[53271]: pgmap v306: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-03-11T20:49:08.402 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:08 smithi204 ceph-mon[53271]: Upgrade: Updating mds.foofs.smithi204.evhral 2024-03-11T20:49:08.402 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:08 smithi204 ceph-mon[53271]: Deploying daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:49:08.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:08 smithi083 ceph-mon[66365]: pgmap v306: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-03-11T20:49:08.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:08 smithi083 ceph-mon[66365]: Upgrade: Updating mds.foofs.smithi204.evhral 2024-03-11T20:49:08.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:08 smithi083 ceph-mon[66365]: Deploying daemon mds.foofs.smithi204.evhral on smithi204 2024-03-11T20:49:09.680 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:49:09 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:49:09.248+0000 7fd33327f700 -1 bdev(0x564b03d54000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 2 2024-03-11T20:49:09.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[66361]: 2024-03-11T20:49:09.221+0000 7fb104963700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:49:09.697 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:49:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:49:09.248+0000 7fe518ffa700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-03-11T20:49:09.697 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:49:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:49:09.248+0000 7fe519ffc700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 2 2024-03-11T20:49:09.697 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:49:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:49:09.249+0000 7fe5147f1700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 10 2024-03-11T20:49:09.697 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:49:09 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:49:09.250+0000 7fe5187f9700 -1 bdev(0x5569c62c8000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 16 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: pgmap v307: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: osdmap e147: 8 total, 8 up, 8 in 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: Standby daemon mds.foofs.smithi083.motgne assigned to filesystem foofs as rank 0 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-11T20:49:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:10 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:replay} 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: pgmap v307: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: osdmap e147: 8 total, 8 up, 8 in 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: Standby daemon mds.foofs.smithi083.motgne assigned to filesystem foofs as rank 0 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-11T20:49:10.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:10 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:replay} 2024-03-11T20:49:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:11 smithi204 ceph-mon[53271]: reconnect by client.14554 172.21.15.83:0/914133764 after 0 2024-03-11T20:49:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:11 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:reconnect 2024-03-11T20:49:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:11 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:reconnect} 2024-03-11T20:49:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:11 smithi204 ceph-mon[53271]: reconnect by client.24357 172.21.15.204:0/1037687027 after 0.0139997 2024-03-11T20:49:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:11 smithi204 ceph-mon[53271]: pgmap v309: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 204 B/s wr, 3 op/s 2024-03-11T20:49:11.696 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:11 smithi083 ceph-mon[66365]: reconnect by client.14554 172.21.15.83:0/914133764 after 0 2024-03-11T20:49:11.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:11 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:reconnect 2024-03-11T20:49:11.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:11 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:reconnect} 2024-03-11T20:49:11.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:11 smithi083 ceph-mon[66365]: reconnect by client.24357 172.21.15.204:0/1037687027 after 0.0139997 2024-03-11T20:49:11.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:11 smithi083 ceph-mon[66365]: pgmap v309: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 204 B/s wr, 3 op/s 2024-03-11T20:49:12.679 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:12 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:rejoin 2024-03-11T20:49:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:12 smithi204 ceph-mon[53271]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:rejoin} 2024-03-11T20:49:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:12 smithi204 ceph-mon[53271]: daemon mds.foofs.smithi083.motgne is now active in filesystem foofs as rank 0 2024-03-11T20:49:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:12.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:12 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:rejoin 2024-03-11T20:49:12.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:12 smithi083 ceph-mon[66365]: fsmap foofs:1/1 {0=foofs.smithi083.motgne=up:rejoin} 2024-03-11T20:49:12.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:12 smithi083 ceph-mon[66365]: daemon mds.foofs.smithi083.motgne is now active in filesystem foofs as rank 0 2024-03-11T20:49:12.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:12.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:13.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:13 smithi204 ceph-mon[53271]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-11T20:49:13.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:13 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:active 2024-03-11T20:49:13.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:13 smithi204 ceph-mon[53271]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 2024-03-11T20:49:13.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:13.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:13 smithi204 ceph-mon[53271]: pgmap v310: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 307 B/s wr, 7 op/s 2024-03-11T20:49:13.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:13 smithi083 ceph-mon[66365]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-11T20:49:13.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:13 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377] up:active 2024-03-11T20:49:13.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:13 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 2024-03-11T20:49:13.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:13.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:13 smithi083 ceph-mon[66365]: pgmap v310: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 307 B/s wr, 7 op/s 2024-03-11T20:49:16.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:15 smithi083 ceph-mon[66365]: pgmap v311: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 307 B/s wr, 8 op/s 2024-03-11T20:49:16.311 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:15 smithi204 ceph-mon[53271]: pgmap v311: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 307 B/s wr, 8 op/s 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: pgmap v312: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 7.7 KiB/s wr, 14 op/s 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-11T20:49:17.638 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: Cluster is now healthy 2024-03-11T20:49:17.639 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: mds.? [v2:172.21.15.204:6832/3472211115,v1:172.21.15.204:6833/3472211115] up:boot 2024-03-11T20:49:17.639 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:49:17.639 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:17 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: pgmap v312: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 7.7 KiB/s wr, 14 op/s 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: Cluster is now healthy 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: mds.? [v2:172.21.15.204:6832/3472211115,v1:172.21.15.204:6833/3472211115] up:boot 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: fsmap foofs:1 {0=foofs.smithi083.motgne=up:active} 1 up:standby 2024-03-11T20:49:17.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:17 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:20.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:19 smithi083 ceph-mon[66365]: pgmap v313: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 7.7 KiB/s wr, 14 op/s 2024-03-11T20:49:20.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:20.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:19 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:20.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:19 smithi204 ceph-mon[53271]: pgmap v313: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 7.7 KiB/s wr, 14 op/s 2024-03-11T20:49:20.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:20.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:19 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:22.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:22.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:22.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:21 smithi204 ceph-mon[53271]: pgmap v314: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.6 KiB/s wr, 12 op/s 2024-03-11T20:49:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:21 smithi083 ceph-mon[66365]: pgmap v314: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.6 KiB/s wr, 12 op/s 2024-03-11T20:49:24.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:23 smithi204 ceph-mon[53271]: pgmap v315: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 7.4 KiB/s wr, 9 op/s 2024-03-11T20:49:24.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:23 smithi083 ceph-mon[66365]: pgmap v315: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 7.4 KiB/s wr, 9 op/s 2024-03-11T20:49:25.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Detected new or changed devices on smithi204 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all mds 2024-03-11T20:49:25.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.motgne"}]': finished 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi204.evhral"}]': finished 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all rgw 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all rbd-mirror 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all ceph-exporter 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.449 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all iscsi 2024-03-11T20:49:25.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.450 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:25 smithi083 ceph-mon[66365]: pgmap v316: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-03-11T20:49:25.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Detected new or changed devices on smithi204 2024-03-11T20:49:25.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:25.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all mds 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.motgne"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.motgne"}]': finished 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi204.evhral"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi204.evhral"}]': finished 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all rgw 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all rbd-mirror 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all ceph-exporter 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all iscsi 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:25.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:25 smithi204 ceph-mon[53271]: pgmap v316: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-03-11T20:49:28.165 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Upgrade: Updating nfs.foo.0.0.smithi083.rbftzq (1/2) 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:49:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Rados config object exists: conf-nfs.foo 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq-rgw 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Bind address in nfs.foo.0.0.smithi083.rbftzq's ganesha conf is defaulting to empty 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: Deploying daemon nfs.foo.0.0.smithi083.rbftzq on smithi083 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:28.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:27 smithi083 ceph-mon[66365]: pgmap v317: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 7.5 KiB/s wr, 5 op/s 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Upgrade: Updating nfs.foo.0.0.smithi083.rbftzq (1/2) 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Rados config object exists: conf-nfs.foo 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Creating key for client.nfs.foo.0.0.smithi083.rbftzq-rgw 2024-03-11T20:49:28.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.rbftzq-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Bind address in nfs.foo.0.0.smithi083.rbftzq's ganesha conf is defaulting to empty 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: Deploying daemon nfs.foo.0.0.smithi083.rbftzq on smithi083 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:28.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:27 smithi204 ceph-mon[53271]: pgmap v317: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 7.5 KiB/s wr, 5 op/s 2024-03-11T20:49:28.848 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (10m) 28s ago 19m 26.9M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 28s ago 19m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (6m) 9s ago 19m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (10m) 28s ago 19m 46.2M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (34s) 28s ago 13m 14.8M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c80d514a96c1 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (12s) 9s ago 13m 12.4M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c21a3f9fbd15 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (9m) 28s ago 21m 511M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (8m) 9s ago 19m 453M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 28s ago 21m 60.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (7m) 9s ago 18m 48.9M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (13m) 28s ago 13m 275M - 4.0 e1d6a67b021e cb4355cdc16f 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (13m) 9s ago 13m 89.9M - 4.0 e1d6a67b021e 32e1a0263caf 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (10m) 28s ago 19m 11.2M - 1.3.1 1dbe0e931976 f0d1b0818508 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (9m) 9s ago 18m 15.0M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:49:28.849 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 28s ago 18m 408M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 28s ago 17m 370M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 28s ago 17m 349M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 28s ago 16m 412M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (2m) 9s ago 16m 348M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (2m) 9s ago 15m 313M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (98s) 9s ago 15m 290M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (60s) 9s ago 15m 291M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:49:28.850 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 28s ago 18m 70.4M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:49:29.179 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:28 smithi204 ceph-mon[53271]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:29.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:28 smithi083 ceph-mon[66365]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:49:29.309 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:49:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:49:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 14 2024-03-11T20:49:29.310 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:49:29.310 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: "mds" 2024-03-11T20:49:29.657 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:49:29.658 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "16/23 daemons upgraded", 2024-03-11T20:49:29.658 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading nfs daemons", 2024-03-11T20:49:29.658 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:49:29.658 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:49:29.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:29 smithi083 ceph-mon[66365]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:29.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:29 smithi083 ceph-mon[66365]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:29.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:29 smithi083 ceph-mon[66365]: pgmap v318: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.2 KiB/s wr, 0 op/s 2024-03-11T20:49:29.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:29 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/381326613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:30.070 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:49:30.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:29 smithi204 ceph-mon[53271]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:30.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:29 smithi204 ceph-mon[53271]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:30.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:29 smithi204 ceph-mon[53271]: pgmap v318: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.2 KiB/s wr, 0 op/s 2024-03-11T20:49:30.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:29 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/381326613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:31.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:30 smithi204 ceph-mon[53271]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:31.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:30 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/4180142645' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:49:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:30 smithi083 ceph-mon[66365]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:49:31.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:30 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/4180142645' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:49:32.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:31 smithi204 ceph-mon[53271]: pgmap v319: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.4 KiB/s wr, 1 op/s 2024-03-11T20:49:32.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:31 smithi204 ceph-mon[53271]: Evicting (and blocklisting) client session 14554 (172.21.15.83:0/914133764) 2024-03-11T20:49:32.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:31 smithi204 ceph-mon[53271]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/914133764"}]: dispatch 2024-03-11T20:49:32.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:32.171 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:32.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:31 smithi083 ceph-mon[66365]: pgmap v319: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.4 KiB/s wr, 1 op/s 2024-03-11T20:49:32.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:31 smithi083 ceph-mon[66365]: Evicting (and blocklisting) client session 14554 (172.21.15.83:0/914133764) 2024-03-11T20:49:32.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:31 smithi083 ceph-mon[66365]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/914133764"}]: dispatch 2024-03-11T20:49:32.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:32.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/914133764"}]': finished 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: osdmap e148: 8 total, 8 up, 8 in 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:33.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:32 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/914133764"}]': finished 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: osdmap e148: 8 total, 8 up, 8 in 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:33.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:32 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Upgrade: Updating nfs.foo.1.0.smithi204.brmoam (2/2) 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Creating key for client.nfs.foo.1.0.smithi204.brmoam 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: pgmap v321: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 921 B/s wr, 2 op/s 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Rados config object exists: conf-nfs.foo 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Creating key for client.nfs.foo.1.0.smithi204.brmoam-rgw 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Bind address in nfs.foo.1.0.smithi204.brmoam's ganesha conf is defaulting to empty 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:34.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:33 smithi204 ceph-mon[53271]: Deploying daemon nfs.foo.1.0.smithi204.brmoam on smithi204 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Upgrade: Updating nfs.foo.1.0.smithi204.brmoam (2/2) 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Creating key for client.nfs.foo.1.0.smithi204.brmoam 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: pgmap v321: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 921 B/s wr, 2 op/s 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-11T20:49:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Rados config object exists: conf-nfs.foo 2024-03-11T20:49:34.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Creating key for client.nfs.foo.1.0.smithi204.brmoam-rgw 2024-03-11T20:49:34.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi204.brmoam-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-11T20:49:34.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Bind address in nfs.foo.1.0.smithi204.brmoam's ganesha conf is defaulting to empty 2024-03-11T20:49:34.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:34.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:33 smithi083 ceph-mon[66365]: Deploying daemon nfs.foo.1.0.smithi204.brmoam on smithi204 2024-03-11T20:49:36.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:35 smithi204 ceph-mon[53271]: pgmap v322: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-11T20:49:36.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:35 smithi083 ceph-mon[66365]: pgmap v322: 97 pgs: 97 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-03-11T20:49:37.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:37.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:37.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:37 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:37.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:37 smithi204 ceph-mon[53271]: pgmap v323: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.2 KiB/s wr, 6 op/s 2024-03-11T20:49:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:37 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:38.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:37 smithi083 ceph-mon[66365]: pgmap v323: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.2 KiB/s wr, 6 op/s 2024-03-11T20:49:40.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:39 smithi083 ceph-mon[66365]: pgmap v324: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 KiB/s wr, 6 op/s 2024-03-11T20:49:40.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:40.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:40.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:39 smithi083 ceph-mon[66365]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.204:0/1037687027"}]: dispatch 2024-03-11T20:49:40.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:39 smithi204 ceph-mon[53271]: pgmap v324: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 KiB/s wr, 6 op/s 2024-03-11T20:49:40.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:40.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:40.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:39 smithi204 ceph-mon[53271]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.204:0/1037687027"}]: dispatch 2024-03-11T20:49:41.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:40 smithi204 ceph-mon[53271]: Evicting (and blocklisting) client session 24357 (172.21.15.204:0/1037687027) 2024-03-11T20:49:41.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:40 smithi204 ceph-mon[53271]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.204:0/1037687027"}]': finished 2024-03-11T20:49:41.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:40 smithi204 ceph-mon[53271]: osdmap e149: 8 total, 8 up, 8 in 2024-03-11T20:49:41.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:41.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:40 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:41.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:40 smithi083 ceph-mon[66365]: Evicting (and blocklisting) client session 24357 (172.21.15.204:0/1037687027) 2024-03-11T20:49:41.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:40 smithi083 ceph-mon[66365]: from='mds.? [v2:172.21.15.83:6834/691281377,v1:172.21.15.83:6835/691281377]' entity='mds.foofs.smithi083.motgne' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.204:0/1037687027"}]': finished 2024-03-11T20:49:41.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:40 smithi083 ceph-mon[66365]: osdmap e149: 8 total, 8 up, 8 in 2024-03-11T20:49:41.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:41.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:40 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:42.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:41 smithi204 ceph-mon[53271]: pgmap v326: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.6 KiB/s wr, 9 op/s 2024-03-11T20:49:42.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:42.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:41 smithi083 ceph-mon[66365]: pgmap v326: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.6 KiB/s wr, 9 op/s 2024-03-11T20:49:42.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:43.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:43 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:43.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:43 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:43.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:43 smithi204 ceph-mon[53271]: pgmap v327: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.5 KiB/s wr, 8 op/s 2024-03-11T20:49:43.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:43.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:43.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:43 smithi083 ceph-mon[66365]: pgmap v327: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.5 KiB/s wr, 8 op/s 2024-03-11T20:49:44.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:44.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:45.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:45 smithi083 ceph-mon[66365]: pgmap v328: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.3 KiB/s wr, 8 op/s 2024-03-11T20:49:45.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:45 smithi204 ceph-mon[53271]: pgmap v328: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.3 KiB/s wr, 8 op/s 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: pgmap v329: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:48.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.rbftzq"}]: dispatch 2024-03-11T20:49:48.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.rbftzq"}]': finished 2024-03-11T20:49:48.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi204.brmoam"}]: dispatch 2024-03-11T20:49:48.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi204.brmoam"}]': finished 2024-03-11T20:49:48.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: pgmap v329: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-03-11T20:49:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.rbftzq"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.rbftzq"}]': finished 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi204.brmoam"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi204.brmoam"}]': finished 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:49:48.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:48 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all nfs 2024-03-11T20:49:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:48 smithi204 ceph-mon[53271]: Upgrade: Setting container_image for all nvmeof 2024-03-11T20:49:49.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:48 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all nfs 2024-03-11T20:49:49.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:48 smithi083 ceph-mon[66365]: Upgrade: Setting container_image for all nvmeof 2024-03-11T20:49:50.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:49 smithi083 ceph-mon[66365]: pgmap v330: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-11T20:49:50.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:49 smithi083 ceph-mon[66365]: Upgrade: Updating node-exporter.smithi083 (1/2) 2024-03-11T20:49:50.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:49 smithi083 ceph-mon[66365]: Deploying daemon node-exporter.smithi083 on smithi083 2024-03-11T20:49:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:49 smithi204 ceph-mon[53271]: pgmap v330: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-11T20:49:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:49 smithi204 ceph-mon[53271]: Upgrade: Updating node-exporter.smithi083 (1/2) 2024-03-11T20:49:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:49 smithi204 ceph-mon[53271]: Deploying daemon node-exporter.smithi083 on smithi083 2024-03-11T20:49:52.231 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:51 smithi083 ceph-mon[66365]: pgmap v331: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 202 B/s rd, 304 B/s wr, 0 op/s 2024-03-11T20:49:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:51 smithi204 ceph-mon[53271]: pgmap v331: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 202 B/s rd, 304 B/s wr, 0 op/s 2024-03-11T20:49:54.232 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:53 smithi204 ceph-mon[53271]: pgmap v332: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:54.377 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:53 smithi083 ceph-mon[66365]: pgmap v332: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:56.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:56 smithi204 ceph-mon[53271]: pgmap v333: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:56.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:56.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:56.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:56 smithi083 ceph-mon[66365]: pgmap v333: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:56.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:56.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:49:57.440 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:57.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:49:58.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:58 smithi083 ceph-mon[66365]: Upgrade: Updating node-exporter.smithi204 (2/2) 2024-03-11T20:49:58.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:58 smithi083 ceph-mon[66365]: pgmap v334: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:58.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:49:58 smithi083 ceph-mon[66365]: Deploying daemon node-exporter.smithi204 on smithi204 2024-03-11T20:49:58.481 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:58 smithi204 ceph-mon[53271]: Upgrade: Updating node-exporter.smithi204 (2/2) 2024-03-11T20:49:58.481 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:58 smithi204 ceph-mon[53271]: pgmap v334: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-03-11T20:49:58.481 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:49:58 smithi204 ceph-mon[53271]: Deploying daemon node-exporter.smithi204 on smithi204 2024-03-11T20:50:00.440 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:50:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:00 smithi083 ceph-mon[66365]: pgmap v335: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:00 smithi083 ceph-mon[66365]: overall HEALTH_OK 2024-03-11T20:50:00.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:00 smithi204 ceph-mon[53271]: pgmap v335: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:00.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:00 smithi204 ceph-mon[53271]: overall HEALTH_OK 2024-03-11T20:50:01.083 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:50:01.083 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (11m) 18s ago 20m 27.0M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:50:01.083 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 18s ago 20m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:50:01.083 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (7m) 21s ago 19m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:50:01.083 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (10m) 18s ago 20m 47.0M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (66s) 18s ago 14m 38.4M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c80d514a96c1 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (45s) 21s ago 14m 12.5M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c21a3f9fbd15 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (10m) 18s ago 21m 513M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (9m) 21s ago 19m 453M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 18s ago 21m 62.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (8m) 21s ago 19m 50.0M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (30s) 18s ago 13m 68.1M - 5.7 4133f8bc41bb dbd24c64d346 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (24s) 21s ago 13m 23.3M - 5.7 4133f8bc41bb 3d491efa78cc 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 starting - - - - 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (10m) 21s ago 19m 15.6M - 1.3.1 1dbe0e931976 2b7d51da5c2d 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 18s ago 18m 409M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 18s ago 18m 372M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:50:01.084 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 18s ago 17m 353M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 18s ago 17m 414M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (3m) 21s ago 16m 348M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (2m) 21s ago 16m 313M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (2m) 21s ago 16m 290M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (92s) 21s ago 15m 291M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:50:01.085 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (9m) 18s ago 19m 70.5M - 2.33.4 514e6a882f6e 06e9913a30a1 2024-03-11T20:50:01.471 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:50:01.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:50:01.471 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 14 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:50:01.472 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "nfs", 2024-03-11T20:50:01.797 INFO:teuthology.orchestra.run.smithi083.stdout: "mds" 2024-03-11T20:50:01.798 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:50:01.798 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "18/23 daemons upgraded", 2024-03-11T20:50:01.798 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading node-exporter daemons", 2024-03-11T20:50:01.798 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:50:01.798 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:50:02.191 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: pgmap v336: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3984324926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:02.386 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:02 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1376155884' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: pgmap v336: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3984324926' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:02 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1376155884' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:50:03.364 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:03 smithi083 ceph-mon[66365]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:03.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:03.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:03.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:03 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:03.504 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:03 smithi204 ceph-mon[53271]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:03.504 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:03.504 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:03.504 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:03 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:04.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:04 smithi204 ceph-mon[53271]: pgmap v337: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:04.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:04 smithi083 ceph-mon[66365]: pgmap v337: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:06.477 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:06 smithi083 ceph-mon[66365]: pgmap v338: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:06.477 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:06.478 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:06.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:06 smithi204 ceph-mon[53271]: pgmap v338: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:06.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:06.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:07.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:07.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:07 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:07.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:07 smithi204 ceph-mon[53271]: pgmap v339: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:07.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:07.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:07 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:07.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:07 smithi083 ceph-mon[66365]: pgmap v339: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:09.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:09.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:09.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:09.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:10.032 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:09 smithi204 ceph-mon[53271]: pgmap v340: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:10.032 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:09 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:10.032 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:09 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:10.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:09 smithi083 ceph-mon[66365]: pgmap v340: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:10.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:09 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:10.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:09 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:11.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:11.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:11 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:11.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:11.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:11 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:12.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:12 smithi204 ceph-mon[53271]: pgmap v341: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:12.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:12.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:12 smithi083 ceph-mon[66365]: pgmap v341: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:12.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:14.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.181 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.182 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:13 smithi204 ceph-mon[53271]: pgmap v342: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:14.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:14.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:14.199 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:13 smithi083 ceph-mon[66365]: pgmap v342: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:16.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:15 smithi204 ceph-mon[53271]: Upgrade: Updating prometheus.smithi083 2024-03-11T20:50:16.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:15 smithi204 ceph-mon[53271]: Deploying daemon prometheus.smithi083 on smithi083 2024-03-11T20:50:16.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:15 smithi204 ceph-mon[53271]: pgmap v343: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:16.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:15 smithi083 ceph-mon[66365]: Upgrade: Updating prometheus.smithi083 2024-03-11T20:50:16.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:15 smithi083 ceph-mon[66365]: Deploying daemon prometheus.smithi083 on smithi083 2024-03-11T20:50:16.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:15 smithi083 ceph-mon[66365]: pgmap v343: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:18.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:17 smithi083 ceph-mon[66365]: pgmap v344: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:18.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:17 smithi204 ceph-mon[53271]: pgmap v344: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:20.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:19 smithi204 ceph-mon[53271]: pgmap v345: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:20.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:19 smithi083 ceph-mon[66365]: pgmap v345: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:21.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:21.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:21 smithi083 ceph-mon[66365]: pgmap v346: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:21.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:21 smithi204 ceph-mon[53271]: pgmap v346: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:24.288 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:23 smithi083 ceph-mon[66365]: pgmap v347: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:24.383 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:23 smithi204 ceph-mon[53271]: pgmap v347: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:26.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:25 smithi204 ceph-mon[53271]: pgmap v348: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:26.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:26.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:25 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:25 smithi083 ceph-mon[66365]: pgmap v348: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:25 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:27.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:27.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:27.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:27.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:27 smithi204 ceph-mon[53271]: pgmap v349: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:27.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:27.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:27.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:27.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:27 smithi083 ceph-mon[66365]: pgmap v349: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:29 smithi204 ceph-mon[53271]: pgmap v350: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:30.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:29 smithi083 ceph-mon[66365]: pgmap v350: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:50:31.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:31 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:31.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:31.432 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:31 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:32.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:32 smithi204 ceph-mon[53271]: pgmap v351: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:32.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:32 smithi083 ceph-mon[66365]: pgmap v351: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:32.557 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:50:33.216 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:50:33.216 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (11m) 7s ago 20m 27.3M - 0.23.0 ba2b418f427c 3e44564e02a2 2024-03-11T20:50:33.216 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 7s ago 20m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:50:33.216 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (7m) 27s ago 20m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:50:33.216 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (11m) 7s ago 20m 51.9M - 8.3.5 dad864ee21e9 de1532b30f47 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (99s) 7s ago 14m 38.6M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c80d514a96c1 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (77s) 27s ago 14m 12.6M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c21a3f9fbd15 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (10m) 7s ago 22m 516M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (9m) 27s ago 20m 453M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 7s ago 22m 63.4M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (8m) 27s ago 20m 51.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (62s) 7s ago 14m 68.4M - 5.7 4133f8bc41bb dbd24c64d346 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (56s) 27s ago 14m 71.5M - 5.7 4133f8bc41bb 3d491efa78cc 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (38s) 7s ago 20m 8514k - 1.5.0 0da6a335fe13 7603f9371334 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (31s) 27s ago 19m 3556k - 1.5.0 0da6a335fe13 461d574109d2 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 7s ago 19m 411M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 7s ago 18m 372M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:50:33.217 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 7s ago 18m 354M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 7s ago 17m 415M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (3m) 27s ago 17m 348M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (3m) 27s ago 17m 314M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (2m) 27s ago 16m 290M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (2m) 27s ago 16m 291M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:50:33.218 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (13s) 7s ago 19m 44.7M - 2.43.0 a07b618ecd1d 869902cfc1d4 2024-03-11T20:50:33.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:33 smithi083 ceph-mon[66365]: Upgrade: Updating alertmanager.smithi083 2024-03-11T20:50:33.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:33 smithi083 ceph-mon[66365]: Deploying daemon alertmanager.smithi083 on smithi083 2024-03-11T20:50:33.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:33 smithi204 ceph-mon[53271]: Upgrade: Updating alertmanager.smithi083 2024-03-11T20:50:33.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:33 smithi204 ceph-mon[53271]: Deploying daemon alertmanager.smithi083 on smithi083 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:50:33.613 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:50:33.614 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:50:33.614 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:50:33.614 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 14 2024-03-11T20:50:33.614 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:50:33.614 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-03-11T20:50:33.949 INFO:teuthology.orchestra.run.smithi083.stdout: "mon", 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "nfs", 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "mds" 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "18/23 daemons upgraded", 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading alertmanager daemons", 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:50:33.950 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:50:34.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:34 smithi083 ceph-mon[66365]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:34 smithi083 ceph-mon[66365]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:34 smithi083 ceph-mon[66365]: pgmap v352: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:34.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:34 smithi083 ceph-mon[66365]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:34 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1643721305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:34.352 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:50:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:34 smithi204 ceph-mon[53271]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:34 smithi204 ceph-mon[53271]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:34 smithi204 ceph-mon[53271]: pgmap v352: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:34 smithi204 ceph-mon[53271]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:34 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1643721305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:35.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:35 smithi083 ceph-mon[66365]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:35.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:35 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1341151249' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:50:35.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:35 smithi204 ceph-mon[53271]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:50:35.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:35 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1341151249' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:50:36.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:36 smithi204 ceph-mon[53271]: pgmap v353: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:36.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:36 smithi083 ceph-mon[66365]: pgmap v353: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:38 smithi204 ceph-mon[53271]: pgmap v354: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:38 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:38 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:38 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:38.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:38 smithi083 ceph-mon[66365]: pgmap v354: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:38.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:38 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:38.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:38 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:38.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:38 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:39.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:39.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:39 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:39.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:39 smithi083 ceph-mon[66365]: pgmap v355: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:39 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:39.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:39 smithi204 ceph-mon[53271]: pgmap v355: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:42.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:41 smithi083 ceph-mon[66365]: pgmap v356: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:42.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:41 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:41 smithi204 ceph-mon[53271]: pgmap v356: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:41 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:43 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:43 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:44.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:43 smithi204 ceph-mon[53271]: pgmap v357: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:43 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:44.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:43 smithi083 ceph-mon[66365]: pgmap v357: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:44 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:45.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:44 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:46.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:45 smithi204 ceph-mon[53271]: pgmap v358: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:46.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:45 smithi083 ceph-mon[66365]: pgmap v358: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:48.282 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: pgmap v359: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.283 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.284 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:47 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: pgmap v359: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:48.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:47 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:50:49.349 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:48 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:50:49.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:48 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-11T20:50:50.232 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:49 smithi083 ceph-mon[66365]: pgmap v360: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:49 smithi204 ceph-mon[53271]: pgmap v360: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:51.233 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:50 smithi083 ceph-mon[66365]: Upgrade: Updating grafana.smithi083 2024-03-11T20:50:51.233 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:50 smithi083 ceph-mon[66365]: Deploying daemon grafana.smithi083 on smithi083 2024-03-11T20:50:51.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:50 smithi204 ceph-mon[53271]: Upgrade: Updating grafana.smithi083 2024-03-11T20:50:51.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:50 smithi204 ceph-mon[53271]: Deploying daemon grafana.smithi083 on smithi083 2024-03-11T20:50:52.076 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:51 smithi083 ceph-mon[66365]: pgmap v361: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:51 smithi204 ceph-mon[53271]: pgmap v361: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:54.237 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:53 smithi083 ceph-mon[66365]: pgmap v362: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:54.251 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:53 smithi204 ceph-mon[53271]: pgmap v362: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:55.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:55.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:55.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:55 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:55.448 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:55 smithi083 ceph-mon[66365]: pgmap v363: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:55.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:55.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:50:55.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:55 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:50:55.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:55 smithi204 ceph-mon[53271]: pgmap v363: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:50:56.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:56 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:57.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:56 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:50:58.323 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:57 smithi083 ceph-mon[66365]: pgmap v364: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:50:58.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:57 smithi204 ceph-mon[53271]: pgmap v364: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:00.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:59 smithi204 ceph-mon[53271]: pgmap v365: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:00.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:59 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:00.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:50:59 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:59 smithi083 ceph-mon[66365]: pgmap v365: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:59 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:50:59 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:01.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:01.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:01 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:01.947 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:01 smithi083 ceph-mon[66365]: pgmap v366: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:02.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:02.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:01 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:02.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:01 smithi204 ceph-mon[53271]: pgmap v366: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:04.237 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:03 smithi083 ceph-mon[66365]: pgmap v367: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:04.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:03 smithi204 ceph-mon[53271]: pgmap v367: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:05.475 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph orch ps' 2024-03-11T20:51:05.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.681 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.682 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: Upgrade: Finalizing container_image settings 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-11T20:51:05.683 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-11T20:51:05.684 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: Upgrade: Complete! 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.685 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:51:05.686 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:05 smithi204 ceph-mon[53271]: pgmap v368: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:05.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: Upgrade: Finalizing container_image settings 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-11T20:51:05.699 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-11T20:51:05.700 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-11T20:51:05.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-11T20:51:05.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-11T20:51:05.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-11T20:51:05.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: Upgrade: Complete! 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-11T20:51:05.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:51:05.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:05 smithi083 ceph-mon[66365]: pgmap v368: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:51:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (30s) 7s ago 21m 16.5M - 0.25.0 c8568f914cd2 ac865f8a0593 2024-03-11T20:51:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 7s ago 21m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:51:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (8m) 61s ago 20m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:51:07.482 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (13s) 7s ago 21m 53.4M - 9.4.12 a0ea5e3c46d9 f260b12ecae8 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (2m) 7s ago 15m 38.9M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c80d514a96c1 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (111s) 61s ago 15m 12.6M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c21a3f9fbd15 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (11m) 7s ago 22m 517M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (10m) 61s ago 20m 453M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 7s ago 22m 64.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (9m) 61s ago 20m 51.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (96s) 7s ago 15m 70.3M - 5.7 4133f8bc41bb dbd24c64d346 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (91s) 61s ago 14m 71.5M - 5.7 4133f8bc41bb 3d491efa78cc 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (72s) 7s ago 21m 8326k - 1.5.0 0da6a335fe13 7603f9371334 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (65s) 61s ago 20m 3556k - 1.5.0 0da6a335fe13 461d574109d2 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 7s ago 19m 411M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 7s ago 19m 372M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 7s ago 18m 354M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 7s ago 18m 415M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (4m) 61s ago 18m 348M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:51:07.483 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (3m) 61s ago 17m 314M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:51:07.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (3m) 61s ago 17m 290M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:51:07.484 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (2m) 61s ago 16m 291M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:51:07.484 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (47s) 7s ago 20m 55.5M - 2.43.0 a07b618ecd1d 869902cfc1d4 2024-03-11T20:51:08.109 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph versions' 2024-03-11T20:51:08.415 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:08 smithi083 ceph-mon[66365]: pgmap v369: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:08.415 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:08 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:08.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:08 smithi204 ceph-mon[53271]: pgmap v369: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:08.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:08 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:51:09.388 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:09 smithi083 ceph-mon[66365]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:51:09.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:09 smithi204 ceph-mon[53271]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:51:10.372 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:51:10.373 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:51:10.374 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 14 2024-03-11T20:51:10.374 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:51:10.374 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:51:10.374 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:10 smithi083 ceph-mon[66365]: pgmap v370: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:10.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:10 smithi204 ceph-mon[53271]: pgmap v370: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:11.033 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-03-11T20:51:11.340 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:11 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1595229800' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:11.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:11 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1595229800' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:51:12.484 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:12 smithi083 ceph-mon[66365]: pgmap v371: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:12.484 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:12 smithi204 ceph-mon[53271]: pgmap v371: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:12.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:12.770 INFO:teuthology.orchestra.run.smithi083.stdout:wait for servicemap items w/ changing names to refresh 2024-03-11T20:51:13.511 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'sleep 60' 2024-03-11T20:51:14.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:14 smithi083 ceph-mon[66365]: pgmap v372: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:14.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:14 smithi204 ceph-mon[53271]: pgmap v372: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:16.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:16 smithi204 ceph-mon[53271]: pgmap v373: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:16.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:16 smithi083 ceph-mon[66365]: pgmap v373: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:18.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:18 smithi204 ceph-mon[53271]: pgmap v374: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:18.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:18 smithi083 ceph-mon[66365]: pgmap v374: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:20.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:20 smithi204 ceph-mon[53271]: pgmap v375: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:20.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:20 smithi083 ceph-mon[66365]: pgmap v375: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:22.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:22 smithi204 ceph-mon[53271]: pgmap v376: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:22.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:22 smithi083 ceph-mon[66365]: pgmap v376: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:23.680 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:23 smithi204 ceph-mon[53271]: pgmap v377: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:23.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:23 smithi083 ceph-mon[66365]: pgmap v377: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:26.383 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:25 smithi204 ceph-mon[53271]: pgmap v378: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:25 smithi083 ceph-mon[66365]: pgmap v378: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:27.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:26 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:27.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:26 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:28.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:27 smithi204 ceph-mon[53271]: pgmap v379: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:28.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:27 smithi083 ceph-mon[66365]: pgmap v379: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:29 smithi204 ceph-mon[53271]: pgmap v380: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:30.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:29 smithi083 ceph-mon[66365]: pgmap v380: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:32.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:31 smithi204 ceph-mon[53271]: pgmap v381: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:32.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:31 smithi083 ceph-mon[66365]: pgmap v381: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:33 smithi204 ceph-mon[53271]: pgmap v382: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:33 smithi083 ceph-mon[66365]: pgmap v382: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:36.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:36 smithi204 ceph-mon[53271]: pgmap v383: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:36.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:36 smithi083 ceph-mon[66365]: pgmap v383: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:38 smithi204 ceph-mon[53271]: pgmap v384: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:38.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:38 smithi083 ceph-mon[66365]: pgmap v384: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:40 smithi204 ceph-mon[53271]: pgmap v385: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:40.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:40 smithi083 ceph-mon[66365]: pgmap v385: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:42 smithi204 ceph-mon[53271]: pgmap v386: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:42.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:42 smithi083 ceph-mon[66365]: pgmap v386: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:42.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:44.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:44 smithi204 ceph-mon[53271]: pgmap v387: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:44.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:44 smithi083 ceph-mon[66365]: pgmap v387: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:46.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:46 smithi204 ceph-mon[53271]: pgmap v388: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:46.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:46 smithi083 ceph-mon[66365]: pgmap v388: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:48.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:48 smithi204 ceph-mon[53271]: pgmap v389: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:48.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:48 smithi083 ceph-mon[66365]: pgmap v389: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:50.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:50 smithi204 ceph-mon[53271]: pgmap v390: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:50.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:50 smithi083 ceph-mon[66365]: pgmap v390: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:52.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:52 smithi204 ceph-mon[53271]: pgmap v391: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:52.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:52 smithi083 ceph-mon[66365]: pgmap v391: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:54.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:54 smithi204 ceph-mon[53271]: pgmap v392: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:54.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:54 smithi083 ceph-mon[66365]: pgmap v392: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:56.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:56 smithi204 ceph-mon[53271]: pgmap v393: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:56.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:56 smithi083 ceph-mon[66365]: pgmap v393: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:51:57.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:57 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:57.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:57 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:51:58.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:51:58 smithi204 ceph-mon[53271]: pgmap v394: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:51:58.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:51:58 smithi083 ceph-mon[66365]: pgmap v394: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:00.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:00 smithi204 ceph-mon[53271]: pgmap v395: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:00.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:00 smithi083 ceph-mon[66365]: pgmap v395: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:02.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:02 smithi204 ceph-mon[53271]: pgmap v396: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:02 smithi083 ceph-mon[66365]: pgmap v396: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:04.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:04 smithi204 ceph-mon[53271]: pgmap v397: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:04.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:04 smithi083 ceph-mon[66365]: pgmap v397: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:05.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:05 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:52:05.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:05 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-11T20:52:06.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:06 smithi204 ceph-mon[53271]: pgmap v398: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:06.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:52:06.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:52:06.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:52:06.431 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:06 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:52:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:06 smithi083 ceph-mon[66365]: pgmap v398: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-11T20:52:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-11T20:52:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:52:06.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:06 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' 2024-03-11T20:52:08.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:08 smithi204 ceph-mon[53271]: pgmap v399: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:08.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:08 smithi083 ceph-mon[66365]: pgmap v399: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:10.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:10 smithi204 ceph-mon[53271]: pgmap v400: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:10.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:10 smithi083 ceph-mon[66365]: pgmap v400: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:12.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:12 smithi204 ceph-mon[53271]: pgmap v401: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:12.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:12 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:12.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:12 smithi083 ceph-mon[66365]: pgmap v401: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:12.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:12 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:14.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:14 smithi204 ceph-mon[53271]: pgmap v402: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:14.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:14 smithi083 ceph-mon[66365]: pgmap v402: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:15.935 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph orch ps' 2024-03-11T20:52:16.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:16 smithi083 ceph-mon[66365]: pgmap v403: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:16.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:16 smithi204 ceph-mon[53271]: pgmap v403: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:17.962 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (100s) 78s ago 22m 16.5M - 0.25.0 c8568f914cd2 ac865f8a0593 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 78s ago 22m 6656k - 19.0.0-1979-g2b0101eb 4133f8bc41bb 7aaf470884b9 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi204 smithi204 running (9m) 2m ago 21m 6660k - 19.0.0-1979-g2b0101eb 4133f8bc41bb ffbe17aa95fa 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (84s) 78s ago 22m 53.4M - 9.4.12 a0ea5e3c46d9 f260b12ecae8 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.motgne smithi083 running (3m) 78s ago 16m 38.9M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c80d514a96c1 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi204.evhral smithi204 running (3m) 2m ago 16m 12.6M - 19.0.0-1979-g2b0101eb 4133f8bc41bb c21a3f9fbd15 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.jyxogi smithi083 *:8443,9283,8765 running (12m) 78s ago 23m 517M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 1572d3dbe0c4 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi204.qknpir smithi204 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-1979-g2b0101eb 4133f8bc41bb 6b02d87a13e6 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (11m) 78s ago 24m 64.3M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 9ee245210402 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi204 smithi204 running (10m) 2m ago 21m 51.1M 2048M 19.0.0-1979-g2b0101eb 4133f8bc41bb 1e6e8a723341 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.rbftzq smithi083 *:2049 running (2m) 78s ago 16m 70.3M - 5.7 4133f8bc41bb dbd24c64d346 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi204.brmoam smithi204 *:2049 running (2m) 2m ago 16m 71.5M - 5.7 4133f8bc41bb 3d491efa78cc 2024-03-11T20:52:17.963 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (2m) 78s ago 22m 8326k - 1.5.0 0da6a335fe13 7603f9371334 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi204 smithi204 *:9100 running (2m) 2m ago 21m 3556k - 1.5.0 0da6a335fe13 461d574109d2 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 78s ago 21m 411M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ec2a49c17fe1 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 78s ago 20m 372M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ccd8e1d55d1f 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 78s ago 20m 354M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb ce3360b00225 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 78s ago 19m 415M 1936M 19.0.0-1979-g2b0101eb 4133f8bc41bb 8ca8907c6ad7 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi204 running (5m) 2m ago 19m 348M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 86259049ea13 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi204 running (5m) 2m ago 18m 314M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb e42c1b3811e4 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi204 running (4m) 2m ago 18m 290M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb 851e2f209f1f 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi204 running (3m) 2m ago 17m 291M 2704M 19.0.0-1979-g2b0101eb 4133f8bc41bb a684bf6fe09f 2024-03-11T20:52:17.964 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (118s) 78s ago 21m 55.5M - 2.43.0 a07b618ecd1d 869902cfc1d4 2024-03-11T20:52:18.261 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:18 smithi083 ceph-mon[66365]: pgmap v404: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:18.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:18 smithi204 ceph-mon[53271]: pgmap v404: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:18.560 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph orch upgrade status' 2024-03-11T20:52:19.131 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:19 smithi083 ceph-mon[66365]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:19.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:19 smithi204 ceph-mon[53271]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:20.372 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:20 smithi083 ceph-mon[66365]: pgmap v405: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:20.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:20 smithi204 ceph-mon[53271]: pgmap v405: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:20.611 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": null, 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": false, 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "", 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": null, 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-03-11T20:52:20.612 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:52:21.269 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph health detail' 2024-03-11T20:52:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:22 smithi083 ceph-mon[66365]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:22.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:22 smithi083 ceph-mon[66365]: pgmap v406: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:22.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:22 smithi204 ceph-mon[53271]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:22.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:22 smithi204 ceph-mon[53271]: pgmap v406: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:23.379 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-03-11T20:52:23.993 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph versions' 2024-03-11T20:52:24.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:24 smithi083 ceph-mon[66365]: pgmap v407: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:24.300 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:24 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/4111196181' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:52:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:24 smithi204 ceph-mon[53271]: pgmap v407: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:24.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:24 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/4111196181' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-11T20:52:26.050 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-03-11T20:52:26.050 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-03-11T20:52:26.050 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:52:26.050 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 8 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 2 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)": 14 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-03-11T20:52:26.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:26 smithi204 ceph-mon[53271]: pgmap v408: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:26.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:26 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/1648962508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:26 smithi083 ceph-mon[66365]: pgmap v408: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:26.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:26 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/1648962508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:26.836 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-03-11T20:52:27.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:27 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:27.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:27 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:28.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:28 smithi083 ceph-mon[66365]: pgmap v409: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:28.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:28 smithi204 ceph-mon[53271]: pgmap v409: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:28.901 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-03-11T20:52:29.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:29 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/2953002244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:29.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:29 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/2953002244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:29.497 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -e sha1=2b0101eb74d60610d544cb8a473e23a35ac1cc0b -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-03-11T20:52:30.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:30 smithi204 ceph-mon[53271]: pgmap v410: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:30.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:30 smithi083 ceph-mon[66365]: pgmap v410: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:31.646 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-1979-g2b0101eb (2b0101eb74d60610d544cb8a473e23a35ac1cc0b) squid (dev)" 2024-03-11T20:52:32.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:32 smithi083 ceph-mon[66365]: pgmap v411: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:32.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:32 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/387040053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:32.288 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-03-11T20:52:32.289 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-03-11T20:52:32.289 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- ceph orch ls -f json 2024-03-11T20:52:32.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:32 smithi204 ceph-mon[53271]: pgmap v411: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:32.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:32 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/387040053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-11T20:52:34.251 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-03-11T20:52:34.251 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-11T20:29:00.938698Z", "last_refresh": "2024-03-11T20:50:59.742854Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-11T20:28:54.561849Z", "last_refresh": "2024-03-11T20:50:05.507131Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-03-11T20:28:57.624896Z", "last_refresh": "2024-03-11T20:50:59.743017Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-11T20:35:41.655535Z", "last_refresh": "2024-03-11T20:50:05.507861Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-11T20:28:52.140668Z", "last_refresh": "2024-03-11T20:50:05.507235Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi204:172.21.15.204=smithi204"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-11T20:29:43.309480Z", "last_refresh": "2024-03-11T20:50:05.507337Z", "running": 2, "size": 2}}, {"events": ["2024-03-11T20:52:05.431473Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-03-11T20:36:00.125242Z", "last_refresh": "2024-03-11T20:50:05.508006Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-11T20:28:59.682526Z", "last_refresh": "2024-03-11T20:50:05.507415Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379", "last_refresh": "2024-03-11T20:50:05.507491Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-11T20:28:56.256944Z", "last_refresh": "2024-03-11T20:50:59.743158Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-11T20:52:34.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:34 smithi204 ceph-mon[53271]: pgmap v412: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:34.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:34 smithi083 ceph-mon[66365]: pgmap v412: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:34.895 INFO:tasks.cephadm:nfs.foo has 2/2 2024-03-11T20:52:34.895 DEBUG:teuthology.parallel:result is None 2024-03-11T20:52:34.895 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-11T20:52:34.906 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:52:34.906 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-03-11T20:52:34.935 INFO:teuthology.orchestra.run.smithi083.stderr:+ umount /mnt/foo 2024-03-11T20:52:34.946 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-11T20:52:34.954 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:52:34.954 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-03-11T20:52:36.229 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:36 smithi083 ceph-mon[66365]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:52:36.229 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:36 smithi083 ceph-mon[66365]: pgmap v413: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:36.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:36 smithi204 ceph-mon[53271]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-11T20:52:36.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:36 smithi204 ceph-mon[53271]: pgmap v413: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:36.962 INFO:teuthology.orchestra.run.smithi083.stdout: "foo" 2024-03-11T20:52:37.653 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'ceph nfs export ls foo --detailed' 2024-03-11T20:52:38.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:38 smithi083 ceph-mon[66365]: pgmap v414: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:38.282 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:38 smithi083 ceph-mon[66365]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:38 smithi204 ceph-mon[53271]: pgmap v414: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:38.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:38 smithi204 ceph-mon[53271]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:39.610 INFO:teuthology.orchestra.run.smithi083.stdout:[ 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "access_type": "RW", 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "clients": [], 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_id": "foo", 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "export_id": 1, 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "fsal": { 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "fs_name": "foofs", 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "CEPH", 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "user_id": "nfs.foo.1" 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "path": "/", 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: "protocols": [ 2024-03-11T20:52:39.611 INFO:teuthology.orchestra.run.smithi083.stdout: 4 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: "pseudo": "/fake", 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: "security_label": true, 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: "squash": "none", 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: "transports": [ 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: "TCP" 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-03-11T20:52:39.612 INFO:teuthology.orchestra.run.smithi083.stdout:] 2024-03-11T20:52:40.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:40 smithi083 ceph-mon[66365]: pgmap v415: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:40.308 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'rados -p .nfs --all ls -' 2024-03-11T20:52:40.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:40 smithi204 ceph-mon[53271]: pgmap v415: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-11T20:52:41.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:41 smithi204 ceph-mon[53271]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:41.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:41 smithi083 ceph-mon[66365]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-11T20:52:42.154 INFO:teuthology.orchestra.run.smithi083.stdout:foo rec-0000000000000004:nfs.foo.1 2024-03-11T20:52:42.155 INFO:teuthology.orchestra.run.smithi083.stdout:foo grace 2024-03-11T20:52:42.155 INFO:teuthology.orchestra.run.smithi083.stdout:foo rec-0000000000000004:nfs.foo.0 2024-03-11T20:52:42.155 INFO:teuthology.orchestra.run.smithi083.stdout:foo conf-nfs.foo 2024-03-11T20:52:42.155 INFO:teuthology.orchestra.run.smithi083.stdout:foo export-1 2024-03-11T20:52:42.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:42 smithi083 ceph-mon[66365]: pgmap v416: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:42.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:42 smithi083 ceph-mon[66365]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:42 smithi204 ceph-mon[53271]: pgmap v416: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:42.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:42 smithi204 ceph-mon[53271]: from='mgr.24621 172.21.15.83:0/1379737797' entity='mgr.smithi083.jyxogi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-11T20:52:42.807 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-11T20:52:42.817 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:52:42.818 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 -- bash -c 'set -ex 2024-03-11T20:52:42.818 DEBUG:teuthology.orchestra.run.smithi083:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-03-11T20:52:42.818 DEBUG:teuthology.orchestra.run.smithi083:> ' 2024-03-11T20:52:43.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:43 smithi083 ceph-mon[66365]: mgrmap e44: smithi083.jyxogi(active, since 11m), standbys: smithi204.qknpir 2024-03-11T20:52:43.816 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:43 smithi083 ceph-mon[66365]: pgmap v417: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:43.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:43 smithi204 ceph-mon[53271]: mgrmap e44: smithi083.jyxogi(active, since 11m), standbys: smithi204.qknpir 2024-03-11T20:52:43.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:43 smithi204 ceph-mon[53271]: pgmap v417: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:44.535 INFO:teuthology.orchestra.run.smithi083.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-03-11T20:52:44.766 INFO:teuthology.orchestra.run.smithi083.stderr:+ [[ 6 -gt 2 ]] 2024-03-11T20:52:45.114 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:44 smithi083 ceph-mon[66365]: from='client.? 172.21.15.83:0/3437272491' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-11T20:52:45.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:44 smithi204 ceph-mon[53271]: from='client.? 172.21.15.83:0/3437272491' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-11T20:52:45.422 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-11T20:52:45.434 INFO:tasks.cephadm:Teardown begin 2024-03-11T20:52:45.435 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:52:45.464 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:52:45.500 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-11T20:52:45.501 DEBUG:teuthology.orchestra.run.smithi083:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-11T20:52:45.520 DEBUG:teuthology.orchestra.run.smithi204:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-11T20:52:45.558 INFO:tasks.cephadm:Stopping all daemons... 2024-03-11T20:52:45.558 INFO:tasks.cephadm.mon.smithi083:Stopping mon.smithi083... 2024-03-11T20:52:45.558 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083 2024-03-11T20:52:45.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:45 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:52:46.180 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:45 smithi204 ceph-mon[53271]: pgmap v418: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:46.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:45 smithi083 ceph-mon[66365]: pgmap v418: 97 pgs: 97 active+clean; 227 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-03-11T20:52:46.197 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:45 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[66361]: 2024-03-11T20:52:45.872+0000 7fb10a16e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:52:46.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:45 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083[66361]: 2024-03-11T20:52:45.872+0000 7fb10a16e700 -1 mon.smithi083@0(leader) e3 *** Got Signal Terminated *** 2024-03-11T20:52:46.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:45 smithi083 podman[110883]: 2024-03-11 20:52:45.945072203 +0000 UTC m=+0.193880018 container died 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-11T20:52:46.570 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:46 smithi083 podman[110883]: 2024-03-11 20:52:46.340579105 +0000 UTC m=+0.589386930 container cleanup 9ee245210402bb6bc5a2e5b27c2b3aa30c0c34296eeaeda103c2f5b939418d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-03-11T20:52:46.571 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Mar 11 20:52:46 smithi083 bash[110883]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi083 2024-03-11T20:52:46.713 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi083.service' 2024-03-11T20:52:46.763 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:52:46.763 INFO:tasks.cephadm.mon.smithi083:Stopped mon.smithi083 2024-03-11T20:52:46.763 INFO:tasks.cephadm.mon.smithi204:Stopping mon.smithi204... 2024-03-11T20:52:46.763 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi204 2024-03-11T20:52:47.101 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:46 smithi204 systemd[1]: Stopping Ceph mon.smithi204 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:52:47.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204[53267]: 2024-03-11T20:52:47.104+0000 7ff9c8e08700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi204 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:52:47.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204[53267]: 2024-03-11T20:52:47.104+0000 7ff9c8e08700 -1 mon.smithi204@1(peon) e3 *** Got Signal Terminated *** 2024-03-11T20:52:47.430 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:47 smithi204 podman[78733]: 2024-03-11 20:52:47.289272297 +0000 UTC m=+0.324204226 container died 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-11T20:52:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:47 smithi204 podman[78733]: 2024-03-11 20:52:47.74317874 +0000 UTC m=+0.778110679 container cleanup 1e6e8a723341f9797ce00693efab03bdc85065ef82fde4c2cd3cc5858bfb2c46 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-2b0101e) 2024-03-11T20:52:47.930 INFO:journalctl@ceph.mon.smithi204.smithi204.stdout:Mar 11 20:52:47 smithi204 bash[78733]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-mon-smithi204 2024-03-11T20:52:48.208 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@mon.smithi204.service' 2024-03-11T20:52:48.257 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:52:48.258 INFO:tasks.cephadm.mon.smithi204:Stopped mon.smithi204 2024-03-11T20:52:48.258 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-11T20:52:48.258 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.0 2024-03-11T20:52:48.583 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:48 smithi083 systemd[1]: Stopping Ceph osd.0 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:52:48.947 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:52:48.586+0000 7f77da423700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:52:48.947 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:52:48.586+0000 7f77da423700 -1 osd.0 149 *** Got signal Terminated *** 2024-03-11T20:52:48.948 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:48 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0[72380]: 2024-03-11T20:52:48.586+0000 7f77da423700 -1 osd.0 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:52:54.442 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:54 smithi083 podman[111014]: 2024-03-11 20:52:54.174386002 +0000 UTC m=+5.717333784 container died ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux ) 2024-03-11T20:52:54.697 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:54 smithi083 podman[111014]: 2024-03-11 20:52:54.545101807 +0000 UTC m=+6.088049586 container cleanup ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-03-11T20:52:54.698 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:54 smithi083 bash[111014]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0 2024-03-11T20:52:55.155 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:54 smithi083 podman[111033]: 2024-03-11 20:52:54.821814638 +0000 UTC m=+0.648518325 container remove ec2a49c17fe111e983ead68cf29a2d4aabe57d8b5db9e62fdf7355c1daac5245 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307) 2024-03-11T20:52:55.447 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.063565693 +0000 UTC m=+0.019213320 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:52:55.447 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.223032447 +0000 UTC m=+0.178680095 container create 3be756df9aaf217182e31c4eae6fc99828baceda8ecc5b3a8f4d17a72a1abbc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-11T20:52:55.800 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.509332163 +0000 UTC m=+0.464979766 container init 3be756df9aaf217182e31c4eae6fc99828baceda8ecc5b3a8f4d17a72a1abbc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-11T20:52:55.800 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.51291602 +0000 UTC m=+0.468563624 container start 3be756df9aaf217182e31c4eae6fc99828baceda8ecc5b3a8f4d17a72a1abbc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307) 2024-03-11T20:52:55.801 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.617511158 +0000 UTC m=+0.573158773 container attach 3be756df9aaf217182e31c4eae6fc99828baceda8ecc5b3a8f4d17a72a1abbc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-11T20:52:56.166 INFO:journalctl@ceph.osd.0.smithi083.stdout:Mar 11 20:52:55 smithi083 podman[111121]: 2024-03-11 20:52:55.804809538 +0000 UTC m=+0.760457146 container died 3be756df9aaf217182e31c4eae6fc99828baceda8ecc5b3a8f4d17a72a1abbc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-0-deactivate, RELEASE=main-2b0101e, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-11T20:52:56.482 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.0.service' 2024-03-11T20:52:56.523 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:52:56.524 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-11T20:52:56.524 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-11T20:52:56.524 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.1 2024-03-11T20:52:56.853 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:52:56 smithi083 systemd[1]: Stopping Ceph osd.1 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:52:57.197 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:52:56 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:52:56.856+0000 7fe535a38700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:52:57.197 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:52:56 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:52:56.856+0000 7fe535a38700 -1 osd.1 149 *** Got signal Terminated *** 2024-03-11T20:52:57.197 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:52:56 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1[75089]: 2024-03-11T20:52:56.856+0000 7fe535a38700 -1 osd.1 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:02.530 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:02 smithi083 podman[111227]: 2024-03-11 20:53:02.251874134 +0000 UTC m=+5.502011601 container died ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-11T20:53:02.904 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:02 smithi083 podman[111227]: 2024-03-11 20:53:02.607684288 +0000 UTC m=+5.857821755 container cleanup ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-11T20:53:02.904 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:02 smithi083 bash[111227]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1 2024-03-11T20:53:03.198 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:02 smithi083 podman[111245]: 2024-03-11 20:53:02.909086688 +0000 UTC m=+0.650096826 container remove ccd8e1d55d1f6464e3bc2e9ba3ae2a96cec17a84b71760a953a23abcffe877a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e) 2024-03-11T20:53:03.509 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.152518638 +0000 UTC m=+0.018667356 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:03.509 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.268678565 +0000 UTC m=+0.134827270 container create 939d3d8429289d0018eaa44221218044fe289a0cfc8ccd8d87415f352377943a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-11T20:53:03.509 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.513362059 +0000 UTC m=+0.379510766 container init 939d3d8429289d0018eaa44221218044fe289a0cfc8ccd8d87415f352377943a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_BRANCH=HEAD) 2024-03-11T20:53:03.800 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.516547737 +0000 UTC m=+0.382696442 container start 939d3d8429289d0018eaa44221218044fe289a0cfc8ccd8d87415f352377943a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD) 2024-03-11T20:53:03.800 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.621617809 +0000 UTC m=+0.487766526 container attach 939d3d8429289d0018eaa44221218044fe289a0cfc8ccd8d87415f352377943a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.schema-version=1.0) 2024-03-11T20:53:04.096 INFO:journalctl@ceph.osd.1.smithi083.stdout:Mar 11 20:53:03 smithi083 podman[111371]: 2024-03-11 20:53:03.805013815 +0000 UTC m=+0.671162518 container died 939d3d8429289d0018eaa44221218044fe289a0cfc8ccd8d87415f352377943a (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-11T20:53:04.428 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.1.service' 2024-03-11T20:53:04.470 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:04.470 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-11T20:53:04.470 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-11T20:53:04.470 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.2 2024-03-11T20:53:04.697 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:04 smithi083 systemd[1]: Stopping Ceph osd.2 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:05.197 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:04 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:53:04.828+0000 7f10a2490700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:05.198 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:04 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:53:04.828+0000 7f10a2490700 -1 osd.2 149 *** Got signal Terminated *** 2024-03-11T20:53:05.198 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:04 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2[77649]: 2024-03-11T20:53:04.828+0000 7f10a2490700 -1 osd.2 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:10.518 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:10 smithi083 podman[111477]: 2024-03-11 20:53:10.219346653 +0000 UTC m=+5.515902339 container died ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, io.buildah.version=1.33.5, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:53:10.842 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:10 smithi083 podman[111477]: 2024-03-11 20:53:10.603792409 +0000 UTC m=+5.900348095 container cleanup ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=main-2b0101e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:10.842 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:10 smithi083 bash[111477]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2 2024-03-11T20:53:11.143 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:10 smithi083 podman[111494]: 2024-03-11 20:53:10.846713451 +0000 UTC m=+0.631442112 container remove ce3360b0022537a5a0cc43921bc79c1f236d94ba78040a1a218ad0f05997bdf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0) 2024-03-11T20:53:11.448 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.147690638 +0000 UTC m=+0.111708836 container create a1d9ffc01314b0c0058f052ee25c53c16693ce7eb84049eeeb11dc0de6d9e64f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-11T20:53:11.448 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.054126759 +0000 UTC m=+0.018144953 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:11.448 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.366930871 +0000 UTC m=+0.330949073 container init a1d9ffc01314b0c0058f052ee25c53c16693ce7eb84049eeeb11dc0de6d9e64f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307) 2024-03-11T20:53:11.448 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.370016246 +0000 UTC m=+0.334034436 container start a1d9ffc01314b0c0058f052ee25c53c16693ce7eb84049eeeb11dc0de6d9e64f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:11.947 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.458548398 +0000 UTC m=+0.422566598 container attach a1d9ffc01314b0c0058f052ee25c53c16693ce7eb84049eeeb11dc0de6d9e64f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-11T20:53:11.948 INFO:journalctl@ceph.osd.2.smithi083.stdout:Mar 11 20:53:11 smithi083 podman[111587]: 2024-03-11 20:53:11.662515165 +0000 UTC m=+0.626533356 container died a1d9ffc01314b0c0058f052ee25c53c16693ce7eb84049eeeb11dc0de6d9e64f (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, ceph=True) 2024-03-11T20:53:12.312 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.2.service' 2024-03-11T20:53:12.355 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:12.355 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-11T20:53:12.356 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-11T20:53:12.356 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3 2024-03-11T20:53:12.679 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:12 smithi083 systemd[1]: Stopping Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:12.947 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:53:12.682+0000 7f6efafe9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:12.948 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:53:12.682+0000 7f6efafe9700 -1 osd.3 149 *** Got signal Terminated *** 2024-03-11T20:53:12.948 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:12 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:53:12.682+0000 7f6efafe9700 -1 osd.3 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:15.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:15 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:15.604+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:15.931 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:15 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:15.560+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:16.430 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:16 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:16.162+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:16.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:16 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:16.620+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:16.931 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:16 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:16.525+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:17.346 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:17 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:17.172+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:17.447 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:16 smithi083 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3[80395]: 2024-03-11T20:53:16.991+0000 7f6ef75dd700 -1 osd.3 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.184064+0000 front 2024-03-11T20:52:51.183931+0000 (oldest deadline 2024-03-11T20:53:16.484051+0000) 2024-03-11T20:53:17.680 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:17 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:17.580+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:17.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:17 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:17.522+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:17.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:17 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:17.531+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:18.430 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:18 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:18.219+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:18.537 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:18 smithi083 podman[111692]: 2024-03-11 20:53:18.25803628 +0000 UTC m=+5.678494598 container died 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:18.887 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:18 smithi083 podman[111692]: 2024-03-11 20:53:18.640466478 +0000 UTC m=+6.060924833 container cleanup 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.build-date=20240307, GIT_CLEAN=True) 2024-03-11T20:53:18.887 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:18 smithi083 bash[111692]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3 2024-03-11T20:53:18.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:18 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:18.539+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:18.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:18 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:18.474+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:18.931 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:18 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:18.567+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:19.197 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:18 smithi083 podman[111722]: 2024-03-11 20:53:18.891449489 +0000 UTC m=+0.631603850 container remove 8ca8907c6ad7ac63dfc4a564b1019fdc5bbb771f9f75a9295b4d1daa6c25ef40 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:19.518 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:19 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:19.260+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:19.540 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.159208823 +0000 UTC m=+0.018579347 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:19.541 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.309113813 +0000 UTC m=+0.168484337 container create 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-11T20:53:19.541 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.545204578 +0000 UTC m=+0.404575102 container init 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-03-11T20:53:19.842 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.548443054 +0000 UTC m=+0.407813591 container start 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240307, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , ceph=True) 2024-03-11T20:53:19.843 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.628827031 +0000 UTC m=+0.488197565 container attach 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:19.931 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:19 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:19.524+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:19.931 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:19 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:19.522+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:19.932 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:19 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:19.582+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:20.137 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:19 smithi083 podman[111818]: 2024-03-11 20:53:19.847582619 +0000 UTC m=+0.706953150 container died 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:20.388 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:20 smithi083 podman[111844]: 2024-03-11 20:53:20.374413512 +0000 UTC m=+0.521223371 container remove 6b3f4e9a437bb4f633bcd3bfd11035e926da95d92d09487dbc34fcaf9f6fe3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-11T20:53:20.389 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:20 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-11T20:53:20.389 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:20 smithi083 systemd[1]: Stopped Ceph osd.3 for c3f853a8-dfe5-11ee-95c8-87774f69a715. 2024-03-11T20:53:20.389 INFO:journalctl@ceph.osd.3.smithi083.stdout:Mar 11 20:53:20 smithi083 systemd[1]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service: Consumed 1min 36.310s CPU time. 2024-03-11T20:53:20.390 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.3.service' 2024-03-11T20:53:20.431 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:20.432 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-11T20:53:20.432 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-11T20:53:20.432 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.4 2024-03-11T20:53:20.564 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:20 smithi204 systemd[1]: Stopping Ceph osd.4 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:20.565 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:20.268+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:20.565 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:20.500+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:20.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:20.568+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:20.931 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:20.612+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:20.931 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:20.762+0000 7fb074205700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:20.931 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:20.762+0000 7fb074205700 -1 osd.4 149 *** Got signal Terminated *** 2024-03-11T20:53:20.931 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:20 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:20.762+0000 7fb074205700 -1 osd.4 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:21.590 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:21 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:21.308+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:21.590 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:21 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:21.464+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:21.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:21 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:21.609+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:21.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:21 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:21.594+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:22.680 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:22.570+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:22.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:22.349+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:22.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:22.576+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:22.682 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:22 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:22.446+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:23.680 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:23.600+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:23.379+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:23.379+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:23.560+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:23.560+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:23.433+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:23.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:23 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:23.433+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:24.680 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:24.602+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:24.417+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:24.417+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:24.577+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:24.577+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:24.385+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:24.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:24 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:24.385+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:25.680 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4[58213]: 2024-03-11T20:53:25.644+0000 7fb06c7f1700 -1 osd.4 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:53.142454+0000 front 2024-03-11T20:52:53.142426+0000 (oldest deadline 2024-03-11T20:53:15.442175+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:25.533+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:25.533+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:25.404+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:25.404+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:25.399+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:25.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:25 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:25.399+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:26.468 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:26 smithi204 podman[78897]: 2024-03-11 20:53:26.214386039 +0000 UTC m=+5.581977217 container died 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-11T20:53:26.468 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:26.392+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:26.468 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:26.392+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:26.468 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:26.436+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:26.469 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:26.436+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:26.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:26 smithi204 podman[78897]: 2024-03-11 20:53:26.663207048 +0000 UTC m=+6.030798235 container cleanup 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307) 2024-03-11T20:53:26.930 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:26 smithi204 bash[78897]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4 2024-03-11T20:53:26.931 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:26.495+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:26.931 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:26 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:26.495+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:27.259 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:26 smithi204 podman[78948]: 2024-03-11 20:53:26.956721843 +0000 UTC m=+0.741376966 container remove 86259049ea13a50bb8c3c7f7a2793c602eb805c47613830410d74f23baaad07d (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-11T20:53:27.514 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:27.417+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:27.514 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:27.417+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:27.513+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:27.513+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:27.434+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:27 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:27.434+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.167373157 +0000 UTC m=+0.017785803 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.274165786 +0000 UTC m=+0.124578421 container create e0ea490c0983384e7ef49f5f9a6db31a4946b8edae3d6decb927284a2cf9a359 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-11T20:53:27.515 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.51909406 +0000 UTC m=+0.369506705 container init e0ea490c0983384e7ef49f5f9a6db31a4946b8edae3d6decb927284a2cf9a359 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, RELEASE=main-2b0101e, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-11T20:53:27.813 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.522277192 +0000 UTC m=+0.372689832 container start e0ea490c0983384e7ef49f5f9a6db31a4946b8edae3d6decb927284a2cf9a359 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:53:27.813 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.627529821 +0000 UTC m=+0.477942474 container attach e0ea490c0983384e7ef49f5f9a6db31a4946b8edae3d6decb927284a2cf9a359 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:28.102 INFO:journalctl@ceph.osd.4.smithi204.stdout:Mar 11 20:53:27 smithi204 podman[79041]: 2024-03-11 20:53:27.818096067 +0000 UTC m=+0.668508707 container died e0ea490c0983384e7ef49f5f9a6db31a4946b8edae3d6decb927284a2cf9a359 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-4-deactivate, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-03-11T20:53:28.552 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.4.service' 2024-03-11T20:53:28.593 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:28.594 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-11T20:53:28.594 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-11T20:53:28.594 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.5 2024-03-11T20:53:28.656 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:28.448+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:28.657 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:28.448+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:28.657 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:28.393+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:28.657 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:28.393+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:28.657 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:28.514+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:28.657 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:28.514+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:28.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 systemd[1]: Stopping Ceph osd.5 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:29.313 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:28.943+0000 7fd354cc7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:29.313 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:28.943+0000 7fd354cc7700 -1 osd.5 149 *** Got signal Terminated *** 2024-03-11T20:53:29.313 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:28 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:28.943+0000 7fd354cc7700 -1 osd.5 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:29.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:29.404+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:29.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:29.404+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:29.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:29.416+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:29.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:29.416+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:29.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:29.554+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:29.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:29 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:29.554+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:30.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:30.602+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:30.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:30.602+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:30.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:30.424+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:30.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:30.424+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:30.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:30.445+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:30.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:30 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:30.445+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:31.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:31.561+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:31.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:31.561+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:31.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:31.468+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:31.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:31.468+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:31.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:31.430+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:31.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:31.430+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:31.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:31 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:31.430+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:32.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:32.538+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:32.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:32.538+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:32.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:32.425+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:32.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:32.425+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:32.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:32.447+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:32.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:32.447+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:32.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:32 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:32.447+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:33.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:33.489+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:33.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:33.489+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:33.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:33.382+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:49.800864+0000 front 2024-03-11T20:52:49.800879+0000 (oldest deadline 2024-03-11T20:53:15.700698+0000) 2024-03-11T20:53:33.681 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5[60466]: 2024-03-11T20:53:33.382+0000 7fd34d2b3700 -1 osd.5 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:59.102179+0000 front 2024-03-11T20:52:59.102279+0000 (oldest deadline 2024-03-11T20:53:22.602127+0000) 2024-03-11T20:53:33.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:33.444+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:33.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:33.444+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:33.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:33 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:33.444+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:34.641 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:34 smithi204 podman[79159]: 2024-03-11 20:53:34.346420668 +0000 UTC m=+5.509738753 container died e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-11T20:53:34.672 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:34.466+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:34.672 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:34.466+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:34.672 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:34.466+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:34.672 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:34.472+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:34.672 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:34.472+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:34.673 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:34 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:34.472+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:34.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:34 smithi204 podman[79159]: 2024-03-11 20:53:34.718625885 +0000 UTC m=+5.881943976 container cleanup e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-11T20:53:34.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:34 smithi204 bash[79159]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5 2024-03-11T20:53:35.318 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:34 smithi204 podman[79177]: 2024-03-11 20:53:34.986951605 +0000 UTC m=+0.642303514 container remove e42c1b3811e48655dc8160d45daf108c55a0caa349f85cddc87159ecf6b28020 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5, org.label-schema.license=GPLv2, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:35.440+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:35.440+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:35.440+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:35.497+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:35.497+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:35.628 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:35 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:35.497+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:35.629 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.321156212 +0000 UTC m=+0.110252181 container create 670ab355a84b7bed4a9dc09e2711450a859e2a3d8f47864fff2ff86efd256f35 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240307, GIT_BRANCH=HEAD, RELEASE=main-2b0101e) 2024-03-11T20:53:35.629 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.228731385 +0000 UTC m=+0.017827326 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:35.629 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.548507931 +0000 UTC m=+0.337603854 container init 670ab355a84b7bed4a9dc09e2711450a859e2a3d8f47864fff2ff86efd256f35 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-2b0101e, org.label-schema.license=GPLv2) 2024-03-11T20:53:35.629 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.551690922 +0000 UTC m=+0.340786855 container start 670ab355a84b7bed4a9dc09e2711450a859e2a3d8f47864fff2ff86efd256f35 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-11T20:53:35.629 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.631856787 +0000 UTC m=+0.420952710 container attach 670ab355a84b7bed4a9dc09e2711450a859e2a3d8f47864fff2ff86efd256f35 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-11T20:53:35.930 INFO:journalctl@ceph.osd.5.smithi204.stdout:Mar 11 20:53:35 smithi204 podman[79267]: 2024-03-11 20:53:35.845756804 +0000 UTC m=+0.634852739 container died 670ab355a84b7bed4a9dc09e2711450a859e2a3d8f47864fff2ff86efd256f35 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, ceph=True, org.label-schema.build-date=20240307, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-11T20:53:36.516 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.5.service' 2024-03-11T20:53:36.601 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:36.601 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-11T20:53:36.602 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-11T20:53:36.602 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.6 2024-03-11T20:53:36.666 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.409+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:36.667 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.409+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:36.667 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.409+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:36.667 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:36.466+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:36.667 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:36.466+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:36.667 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:36.466+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:36.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 systemd[1]: Stopping Ceph osd.6 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.939+0000 7f4f1d0a4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.939+0000 7f4f1d0a4700 -1 osd.6 149 *** Got signal Terminated *** 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:36 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:36.939+0000 7f4f1d0a4700 -1 osd.6 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:37.387+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:37.387+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:37.384 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:37.387+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:37.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:37.459+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:37.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:37.459+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:37.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:37 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:37.459+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:38.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:38.427+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:38.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:38.427+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:38.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:38.427+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:38.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:38.507+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:38.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:38.507+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:38.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:38 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:38.507+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:39.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:39.379+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:39.379+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:39.379+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:39.522+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:39.522+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:39.522+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:39.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:39 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:39.522+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:40.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:40.400+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:40.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:40.400+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:40.680 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:40.400+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:40.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:40.400+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:17.389769+0000 front 2024-03-11T20:53:17.389740+0000 (oldest deadline 2024-03-11T20:53:39.689522+0000) 2024-03-11T20:53:40.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:40.513+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:40.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:40.513+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:40.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:40.513+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:40.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:40 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:40.513+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:41.422+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:51.686132+0000 front 2024-03-11T20:52:51.686322+0000 (oldest deadline 2024-03-11T20:53:17.486707+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:41.422+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:52:57.487355+0000 front 2024-03-11T20:52:57.487421+0000 (oldest deadline 2024-03-11T20:53:22.787271+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:41.422+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:08.088350+0000 front 2024-03-11T20:53:08.088303+0000 (oldest deadline 2024-03-11T20:53:33.988155+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6[62732]: 2024-03-11T20:53:41.422+0000 7f4f15690700 -1 osd.6 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:17.389769+0000 front 2024-03-11T20:53:17.389740+0000 (oldest deadline 2024-03-11T20:53:39.689522+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:41.476+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:41.476+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:41.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:41.476+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:41.682 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:41 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:41.476+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:42.603 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:42 smithi204 podman[79373]: 2024-03-11 20:53:42.323431055 +0000 UTC m=+5.482569077 container died 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, RELEASE=main-2b0101e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:42.603 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:42 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:42.463+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:42.603 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:42 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:42.463+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:42.603 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:42 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:42.463+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:42.603 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:42 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:42.463+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:42.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:42 smithi204 podman[79373]: 2024-03-11 20:53:42.773064775 +0000 UTC m=+5.932202797 container cleanup 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-11T20:53:42.930 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:42 smithi204 bash[79373]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6 2024-03-11T20:53:43.421 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79392]: 2024-03-11 20:53:43.091556211 +0000 UTC m=+0.762906470 container remove 851e2f209f1f0f1def16fc482296393114c83d21f6f04f07443ee8939fe88415 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, RELEASE=main-2b0101e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-11T20:53:43.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:43.451+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:43.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:43.451+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:43.451+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:43 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:43.451+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.425898396 +0000 UTC m=+0.106210874 container create 65ce4d111046630b3ab4a785fcfb0891609409b75d55f907943436ba8e33e827 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, io.buildah.version=1.33.5, org.label-schema.build-date=20240307, ceph=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.337416877 +0000 UTC m=+0.017729360 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.628343633 +0000 UTC m=+0.308656112 container init 65ce4d111046630b3ab4a785fcfb0891609409b75d55f907943436ba8e33e827 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-2b0101e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30) 2024-03-11T20:53:43.681 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.631413499 +0000 UTC m=+0.311725983 container start 65ce4d111046630b3ab4a785fcfb0891609409b75d55f907943436ba8e33e827 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:44.180 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.736862253 +0000 UTC m=+0.417174743 container attach 65ce4d111046630b3ab4a785fcfb0891609409b75d55f907943436ba8e33e827 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240307, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-11T20:53:44.181 INFO:journalctl@ceph.osd.6.smithi204.stdout:Mar 11 20:53:43 smithi204 podman[79510]: 2024-03-11 20:53:43.929780113 +0000 UTC m=+0.610092595 container died 65ce4d111046630b3ab4a785fcfb0891609409b75d55f907943436ba8e33e827 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240307, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:44.525 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.6.service' 2024-03-11T20:53:44.605 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:44.606 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-11T20:53:44.606 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-11T20:53:44.606 DEBUG:teuthology.orchestra.run.smithi204:> sudo systemctl stop ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.7 2024-03-11T20:53:44.669 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.411+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:44.669 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.411+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:44.669 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.411+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:44.669 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.411+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:44.930 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 systemd[1]: Stopping Ceph osd.7 for c3f853a8-dfe5-11ee-95c8-87774f69a715... 2024-03-11T20:53:45.385 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.934+0000 7fed851a3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.934+0000 7fed851a3700 -1 osd.7 149 *** Got signal Terminated *** 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:44 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:44.934+0000 7fed851a3700 -1 osd.7 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:45.389+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:45.389+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:45.389+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:45.386 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:45 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:45.389+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:46.629 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:46 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:46.346+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:46.629 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:46 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:46.346+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:46.630 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:46 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:46.346+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:46.630 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:46 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:46.346+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:47.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:47.387+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:47.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:47.387+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:47.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:47.387+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:47.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:47 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:47.387+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:48.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:48.390+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:48.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:48.390+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:48.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:48.390+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:48.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:48.390+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:48.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:48 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:48.390+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.204:6804 osd.4 since back 2024-03-11T20:53:24.737689+0000 front 2024-03-11T20:53:24.737533+0000 (oldest deadline 2024-03-11T20:53:47.537923+0000) 2024-03-11T20:53:49.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:49 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:49.372+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-03-11T20:52:52.832758+0000 front 2024-03-11T20:52:52.832957+0000 (oldest deadline 2024-03-11T20:53:15.132906+0000) 2024-03-11T20:53:49.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:49 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:49.372+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-03-11T20:53:01.534381+0000 front 2024-03-11T20:53:01.534428+0000 (oldest deadline 2024-03-11T20:53:23.234287+0000) 2024-03-11T20:53:49.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:49 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:49.372+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-03-11T20:53:07.235686+0000 front 2024-03-11T20:53:07.235806+0000 (oldest deadline 2024-03-11T20:53:30.735697+0000) 2024-03-11T20:53:49.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:49 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:49.372+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-03-11T20:53:13.036607+0000 front 2024-03-11T20:53:13.036593+0000 (oldest deadline 2024-03-11T20:53:38.936396+0000) 2024-03-11T20:53:49.682 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:49 smithi204 ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7[64987]: 2024-03-11T20:53:49.372+0000 7fed7d78f700 -1 osd.7 149 heartbeat_check: no reply from 172.21.15.204:6804 osd.4 since back 2024-03-11T20:53:24.737689+0000 front 2024-03-11T20:53:24.737533+0000 (oldest deadline 2024-03-11T20:53:47.537923+0000) 2024-03-11T20:53:50.723 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:50 smithi204 podman[79615]: 2024-03-11 20:53:50.467047622 +0000 UTC m=+5.639383511 container died a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-11T20:53:51.015 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:50 smithi204 podman[79615]: 2024-03-11 20:53:50.801798134 +0000 UTC m=+5.974134018 container cleanup a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-11T20:53:51.015 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:50 smithi204 bash[79615]: ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7 2024-03-11T20:53:51.413 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79644]: 2024-03-11 20:53:51.019552714 +0000 UTC m=+0.555479559 container remove a684bf6fe09f383230cedecdafecaf58f5370421ee98af37f8a13137e82a4461 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7, GIT_CLEAN=True, org.label-schema.build-date=20240307, CEPH_POINT_RELEASE=, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=main-2b0101e, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-11T20:53:51.680 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.321215273 +0000 UTC m=+0.017827352 image pull 4133f8bc41bb201212a28168427a2edb1f6f3795e694bf61031ea9e2f3ff5e87 quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379 2024-03-11T20:53:51.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.420369151 +0000 UTC m=+0.116981217 container create 0184736c6f566715da4cd387458a1c25e6a3c63a1444475b2bc7d49d5a7067c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, org.label-schema.build-date=20240307, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-2b0101e) 2024-03-11T20:53:51.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.631318763 +0000 UTC m=+0.327930834 container init 0184736c6f566715da4cd387458a1c25e6a3c63a1444475b2bc7d49d5a7067c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240307, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_BRANCH=HEAD) 2024-03-11T20:53:51.681 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.634486437 +0000 UTC m=+0.331098516 container start 0184736c6f566715da4cd387458a1c25e6a3c63a1444475b2bc7d49d5a7067c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-2b0101e, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240307) 2024-03-11T20:53:52.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.731629513 +0000 UTC m=+0.428241599 container attach 0184736c6f566715da4cd387458a1c25e6a3c63a1444475b2bc7d49d5a7067c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, RELEASE=main-2b0101e, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-11T20:53:52.180 INFO:journalctl@ceph.osd.7.smithi204.stdout:Mar 11 20:53:51 smithi204 podman[79739]: 2024-03-11 20:53:51.922516202 +0000 UTC m=+0.619128272 container died 0184736c6f566715da4cd387458a1c25e6a3c63a1444475b2bc7d49d5a7067c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8fe9635f3beb0247496d7e321f3068bd10c333c433ef4824d7e1c1a7d3d78379, name=ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=08b1ab54e11946f162e232e2750e80887689df30, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240307, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-2b0101e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-11T20:53:52.578 DEBUG:teuthology.orchestra.run.smithi204:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c3f853a8-dfe5-11ee-95c8-87774f69a715@osd.7.service' 2024-03-11T20:53:52.662 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-11T20:53:52.662 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-11T20:53:52.663 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 --force --keep-logs 2024-03-11T20:56:50.115 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 --force --keep-logs 2024-03-11T20:57:08.825 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:57:08.862 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-11T20:57:08.891 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-11T20:57:08.893 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi083/crash 2024-03-11T20:57:08.894 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash -- . 2024-03-11T20:57:08.952 INFO:teuthology.orchestra.run.smithi083.stderr:tar: /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash: Cannot open: No such file or directory 2024-03-11T20:57:08.954 INFO:teuthology.orchestra.run.smithi083.stderr:tar: Error is not recoverable: exiting now 2024-03-11T20:57:08.956 DEBUG:teuthology.misc:Transferring archived files from smithi204:/var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi204/crash 2024-03-11T20:57:08.957 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash -- . 2024-03-11T20:57:09.005 INFO:teuthology.orchestra.run.smithi204.stderr:tar: /var/lib/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/crash: Cannot open: No such file or directory 2024-03-11T20:57:09.005 INFO:teuthology.orchestra.run.smithi204.stderr:tar: Error is not recoverable: exiting now 2024-03-11T20:57:09.006 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-11T20:57:09.006 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-11T20:57:09.041 INFO:teuthology.orchestra.run.smithi083.stdout:2024-03-11T20:33:38.064181+0000 mon.smithi083 (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:57:09.041 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-11T20:57:09.041 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[SEC\]' /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-11T20:57:09.112 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]' /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-11T20:57:09.182 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[WRN\]' /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-11T20:57:09.252 INFO:teuthology.orchestra.run.smithi083.stdout:2024-03-11T20:33:38.064181+0000 mon.smithi083 (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-11T20:57:09.253 INFO:tasks.cephadm:Compressing logs... 2024-03-11T20:57:09.253 DEBUG:teuthology.orchestra.run.smithi083:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-11T20:57:09.295 DEBUG:teuthology.orchestra.run.smithi204:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi083.motgne.log 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:: No such file or directory 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi083.log 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi083.jyxogi.log 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi083.motgne.log: /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi083.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log 2024-03-11T20:57:09.325 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi083.jyxogi.log: 89.2% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi083.motgne.log.gz 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.1.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.3.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.0.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.2.log 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log: 85.6% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log.gz 2024-03-11T20:57:09.326 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log 2024-03-11T20:57:09.328 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.2.log: 91.5% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log.gz 2024-03-11T20:57:09.328 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-11T20:57:09.328 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log: 87.0% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log.gz 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.6.log 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:: No such file or directory 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi204.log 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi204.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi204.qknpir.log 2024-03-11T20:57:09.333 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.4.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi204.qknpir.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi204.evhral.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.7.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi204.evhral.log: gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.5.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.7.log: 87.6%gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr: -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mds.foofs.smithi204.evhral.log.gz 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.5.log: /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log: 85.6% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.cephadm.log.gz 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log: 91.6% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.audit.log.gz 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr: 87.0% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph.log.gz 2024-03-11T20:57:09.334 INFO:teuthology.orchestra.run.smithi204.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-11T20:57:09.338 INFO:teuthology.orchestra.run.smithi204.stderr:/var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log: /var/log/ceph/cephadm.log: 90.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-11T20:57:09.344 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/cephadm.log: 91.4% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log.gz 2024-03-11T20:57:09.346 INFO:teuthology.orchestra.run.smithi204.stderr: 91.4% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-volume.log.gz 2024-03-11T20:57:09.346 INFO:teuthology.orchestra.run.smithi083.stderr: 89.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-11T20:57:09.373 INFO:teuthology.orchestra.run.smithi204.stderr: 88.9% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi204.qknpir.log.gz 2024-03-11T20:57:09.673 INFO:teuthology.orchestra.run.smithi204.stderr: 92.7% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi204.log.gz 2024-03-11T20:57:09.713 INFO:teuthology.orchestra.run.smithi083.stderr: 88.2% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mgr.smithi083.jyxogi.log.gz 2024-03-11T20:57:10.664 INFO:teuthology.orchestra.run.smithi083.stderr: 92.0% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-mon.smithi083.log.gz 2024-03-11T20:57:22.002 INFO:teuthology.orchestra.run.smithi083.stderr: 94.5% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.2.log.gz 2024-03-11T20:57:24.593 INFO:teuthology.orchestra.run.smithi204.stderr: 93.9% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.4.log.gz 2024-03-11T20:57:26.225 INFO:teuthology.orchestra.run.smithi204.stderr: 94.2% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.6.log.gz 2024-03-11T20:57:27.089 INFO:teuthology.orchestra.run.smithi083.stderr: 94.1% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.0.log.gz 2024-03-11T20:57:28.130 INFO:teuthology.orchestra.run.smithi083.stderr: 94.2% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.1.log.gz 2024-03-11T20:57:29.002 INFO:teuthology.orchestra.run.smithi204.stderr: 94.2% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.5.log.gz 2024-03-11T20:57:30.264 INFO:teuthology.orchestra.run.smithi083.stderr: 94.1% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.3.log.gz 2024-03-11T20:57:30.266 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-03-11T20:57:30.266 INFO:teuthology.orchestra.run.smithi083.stderr:real 0m20.958s 2024-03-11T20:57:30.266 INFO:teuthology.orchestra.run.smithi083.stderr:user 1m9.096s 2024-03-11T20:57:30.266 INFO:teuthology.orchestra.run.smithi083.stderr:sys 0m2.743s 2024-03-11T20:57:34.522 INFO:teuthology.orchestra.run.smithi204.stderr: 94.1% -- replaced with /var/log/ceph/c3f853a8-dfe5-11ee-95c8-87774f69a715/ceph-osd.7.log.gz 2024-03-11T20:57:34.524 INFO:teuthology.orchestra.run.smithi204.stderr: 2024-03-11T20:57:34.524 INFO:teuthology.orchestra.run.smithi204.stderr:real 0m25.215s 2024-03-11T20:57:34.524 INFO:teuthology.orchestra.run.smithi204.stderr:user 1m14.376s 2024-03-11T20:57:34.524 INFO:teuthology.orchestra.run.smithi204.stderr:sys 0m2.975s 2024-03-11T20:57:34.524 INFO:tasks.cephadm:Archiving logs... 2024-03-11T20:57:34.525 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi083/log 2024-03-11T20:57:34.526 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-11T20:57:44.546 DEBUG:teuthology.misc:Transferring archived files from smithi204:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi204/log 2024-03-11T20:57:44.547 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-11T20:57:55.269 INFO:tasks.cephadm:Removing cluster... 2024-03-11T20:57:55.269 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 --force 2024-03-11T20:57:55.641 DEBUG:teuthology.orchestra.run.smithi204:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c3f853a8-dfe5-11ee-95c8-87774f69a715 --force 2024-03-11T20:57:56.026 INFO:tasks.cephadm:Removing cephadm ... 2024-03-11T20:57:56.026 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-11T20:57:56.041 DEBUG:teuthology.orchestra.run.smithi204:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-11T20:57:56.056 INFO:tasks.cephadm:Teardown complete 2024-03-11T20:57:56.056 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-11T20:57:56.067 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-11T20:57:56.067 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-11T20:57:56.084 DEBUG:teuthology.orchestra.run.smithi204:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-11T20:57:56.095 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 17 +1236us[+1249us] +/- 134ms 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:^* hv02.front.sepia.ceph.com 3 6 377 16 -1733us[-1720us] +/- 73ms 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 +1392us[+1405us] +/- 91ms 2024-03-11T20:57:56.100 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:57:56.109 INFO:teuthology.orchestra.run.smithi204.stderr:bash: line 1: ntpq: command not found 2024-03-11T20:57:56.113 INFO:teuthology.orchestra.run.smithi204.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-11T20:57:56.113 INFO:teuthology.orchestra.run.smithi204.stdout:=============================================================================== 2024-03-11T20:57:56.114 INFO:teuthology.orchestra.run.smithi204.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 17 +1241us[+1241us] +/- 134ms 2024-03-11T20:57:56.114 INFO:teuthology.orchestra.run.smithi204.stdout:^* hv02.front.sepia.ceph.com 3 6 377 18 -1721us[-1684us] +/- 73ms 2024-03-11T20:57:56.114 INFO:teuthology.orchestra.run.smithi204.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 18 +1389us[+1426us] +/- 91ms 2024-03-11T20:57:56.114 INFO:teuthology.orchestra.run.smithi204.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-11T20:57:56.115 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-11T20:57:56.125 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-11T20:57:56.125 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-11T20:57:56.135 DEBUG:teuthology.orchestra.run.smithi083:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-11T20:57:56.223 DEBUG:teuthology.orchestra.run.smithi204:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-11T20:57:56.300 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-11T20:57:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:57:56.331 DEBUG:teuthology.orchestra.run.smithi204:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-11T20:57:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:57:56.361 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-11T20:57:56.371 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-11T20:57:56.381 INFO:teuthology.task.internal:Duration was 2086.789690 seconds 2024-03-11T20:57:56.381 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-11T20:57:56.391 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-11T20:57:56.391 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-11T20:57:56.393 DEBUG:teuthology.orchestra.run.smithi204:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-11T20:57:56.434 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-11T20:57:56.444 INFO:teuthology.orchestra.run.smithi204.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-11T20:57:56.740 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-11T20:57:56.740 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com 2024-03-11T20:57:56.740 DEBUG:teuthology.orchestra.run.smithi083:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-11T20:57:56.801 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi204.front.sepia.ceph.com 2024-03-11T20:57:56.801 DEBUG:teuthology.orchestra.run.smithi204:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-11T20:57:56.826 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-11T20:57:56.826 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-11T20:57:56.843 DEBUG:teuthology.orchestra.run.smithi204:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-11T20:57:56.892 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-11T20:57:56.893 DEBUG:teuthology.orchestra.run.smithi083:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-11T20:57:56.911 DEBUG:teuthology.orchestra.run.smithi204:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-11T20:58:10.468 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-11T20:58:10.480 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-11T20:58:10.480 DEBUG:teuthology.orchestra.run.smithi083:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-11T20:58:10.508 DEBUG:teuthology.orchestra.run.smithi204:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-11T20:58:10.541 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-11T20:58:10.552 DEBUG:teuthology.orchestra.run.smithi083:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-11T20:58:10.555 DEBUG:teuthology.orchestra.run.smithi204:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-11T20:58:10.578 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = core 2024-03-11T20:58:10.610 INFO:teuthology.orchestra.run.smithi204.stdout:kernel.core_pattern = core 2024-03-11T20:58:10.647 DEBUG:teuthology.orchestra.run.smithi083:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-11T20:58:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:58:10.703 DEBUG:teuthology.orchestra.run.smithi204:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-11T20:58:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-11T20:58:10.718 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-11T20:58:10.730 INFO:teuthology.task.internal:Transferring archived files... 2024-03-11T20:58:10.731 DEBUG:teuthology.misc:Transferring archived files from smithi083:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi083 2024-03-11T20:58:10.731 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-11T20:58:10.812 DEBUG:teuthology.misc:Transferring archived files from smithi204:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732/remote/smithi204 2024-03-11T20:58:10.813 DEBUG:teuthology.orchestra.run.smithi204:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-11T20:58:10.857 INFO:teuthology.task.internal:Removing archive directory... 2024-03-11T20:58:10.858 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-11T20:58:10.860 DEBUG:teuthology.orchestra.run.smithi204:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-11T20:58:10.906 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-11T20:58:10.918 INFO:teuthology.task.internal:Not uploading archives. 2024-03-11T20:58:10.919 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-11T20:58:10.929 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-11T20:58:10.930 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-11T20:58:10.932 DEBUG:teuthology.orchestra.run.smithi204:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-11T20:58:10.944 INFO:teuthology.orchestra.run.smithi083.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 11 20:58 /home/ubuntu/cephtest 2024-03-11T20:58:10.961 INFO:teuthology.orchestra.run.smithi204.stdout: 265424 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 11 20:58 /home/ubuntu/cephtest 2024-03-11T20:58:10.962 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-11T20:58:10.973 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-11T20:58:10.996 INFO:teuthology.nuke:Checking targets against current locks 2024-03-11T20:58:11.023 DEBUG:teuthology.nuke:shortname: smithi083 2024-03-11T20:58:11.024 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-11T20:58:11.049 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-11 20:13:13.635943', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDJovG9nEoNTNJG3ojgLf5glFR6waL5+aaAGl1CRgUJ30B8vT/HvrH8awZZ4XqoY4Dl8fYEutRwuFjfaV+cttFk='} 2024-03-11T20:58:11.053 DEBUG:teuthology.nuke:shortname: smithi204 2024-03-11T20:58:11.053 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-11T20:58:11.078 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi204.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-11_20:08:13-orch-main-distro-default-smithi/7591732', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-11 20:13:13.637506', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCXcG/ZdMH6kFIx8fh2ZW0FgkKbnzw3np2nZJt4qUtWpCV68lGgo2UMuTWTXlwB/rL2Az3njM2gLjxLDpvyQWJw='} 2024-03-11T20:58:11.101 INFO:teuthology.orchestra.console.smithi083:Power off 2024-03-11T20:58:11.101 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-11T20:58:11.125 DEBUG:teuthology.orchestra.console.smithi083:power off output: Chassis Power Control: Down/Off 2024-03-11T20:58:11.126 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:11.141 INFO:teuthology.orchestra.console.smithi204:Power off 2024-03-11T20:58:11.142 DEBUG:teuthology.orchestra.console.smithi204:pexpect command: ipmitool -H smithi204.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-11T20:58:11.152 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-03-11T20:58:11.164 DEBUG:teuthology.orchestra.console.smithi204:power off output: Chassis Power Control: Down/Off 2024-03-11T20:58:11.165 DEBUG:teuthology.orchestra.console.smithi204:pexpect command: ipmitool -H smithi204.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:11.188 DEBUG:teuthology.orchestra.console.smithi204:check power output: Chassis Power is on 2024-03-11T20:58:15.153 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:15.189 DEBUG:teuthology.orchestra.console.smithi204:pexpect command: ipmitool -H smithi204.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:15.263 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-03-11T20:58:15.299 DEBUG:teuthology.orchestra.console.smithi204:check power output: Chassis Power is on 2024-03-11T20:58:19.266 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:19.300 DEBUG:teuthology.orchestra.console.smithi204:pexpect command: ipmitool -H smithi204.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:19.379 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is off 2024-03-11T20:58:19.411 DEBUG:teuthology.orchestra.console.smithi204:check power output: Chassis Power is on 2024-03-11T20:58:19.480 INFO:teuthology.orchestra.console.smithi083:Power off completed 2024-03-11T20:58:23.414 DEBUG:teuthology.orchestra.console.smithi204:pexpect command: ipmitool -H smithi204.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-11T20:58:23.524 DEBUG:teuthology.orchestra.console.smithi204:check power output: Chassis Power is off 2024-03-11T20:58:23.625 INFO:teuthology.orchestra.console.smithi204:Power off completed 2024-03-11T20:58:23.728 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2086.789690256119 failure_reason: '"2024-03-11T20:33:38.064181+0000 mon.smithi083 (mon.0) 497 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-03-11T20:58:23.729 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-11T20:58:23.816 INFO:teuthology.run:FAIL